From 6bf7bc963b0dd9b7a6153b675ae46801430dd1b3 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 24 Jan 2024 19:18:41 +0530 Subject: [PATCH 001/133] Split the cluster state remote global metadata file to metadata attribute files Signed-off-by: Shivansh Arora --- .../opensearch/cluster/metadata/Metadata.java | 62 ++- .../metadata/RepositoriesMetadata.java | 4 + .../cluster/metadata/TemplatesMetadata.java | 147 ++++++ .../remote/ClusterMetadataManifest.java | 233 +++++++++- .../remote/RemoteClusterStateService.java | 432 +++++++++++++++++- 5 files changed, 838 insertions(+), 40 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 1871ed24973c2..464a6e89effbd 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -175,6 +175,27 @@ public enum XContentContext { public interface Custom extends NamedDiffable, ToXContentFragment, ClusterState.FeatureAware { EnumSet context(); + + static Custom fromXContent(XContentParser parser, String name) throws IOException { + try { + return parser.namedObject(Custom.class, name, null); + } catch (NamedObjectNotFoundException e) { + logger.warn("Unknown custom object with type {}", name); + parser.skipChildren(); + throw e; + } + } + + static Custom fromXContent(XContentParser parser) throws IOException { + String currentFieldName = parser.currentName(); + try { + return parser.namedObject(Custom.class, currentFieldName, null); + } catch (NamedObjectNotFoundException e) { + logger.warn("Unknown custom object with type {}", currentFieldName); + parser.skipChildren(); + throw e; + } + } } public static final Setting DEFAULT_REPLICA_COUNT_SETTING = Setting.intSetting( @@ -260,7 +281,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust private final Settings settings; private final DiffableStringMap hashesOfConsistentSettings; private final Map indices; - private final Map templates; + private final TemplatesMetadata templates; private final Map customs; private final transient int totalNumberOfShards; // Transient ? not serializable anyway? @@ -304,7 +325,7 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust this.hashesOfConsistentSettings = hashesOfConsistentSettings; this.indices = Collections.unmodifiableMap(indices); this.customs = Collections.unmodifiableMap(customs); - this.templates = Collections.unmodifiableMap(templates); + this.templates = new TemplatesMetadata(templates); int totalNumberOfShards = 0; int totalOpenIndexShards = 0; for (IndexMetadata cursor : indices.values()) { @@ -806,13 +827,17 @@ public Map getIndices() { } public Map templates() { - return this.templates; + return this.templates.getTemplates(); } public Map getTemplates() { return templates(); } + public TemplatesMetadata templatesMetadata() { + return this.templates; + } + public Map componentTemplates() { return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE)) .map(ComponentTemplateMetadata::componentTemplates) @@ -945,7 +970,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) { return false; } - if (!metadata1.templates.equals(metadata2.templates())) { + if (!metadata1.templates.equals(metadata2.templates)) { return false; } // Check if any persistent metadata needs to be saved @@ -1012,7 +1037,7 @@ private static class MetadataDiff implements Diff { persistentSettings = after.persistentSettings; hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings); indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer()); - templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer()); + templates = DiffableUtils.diff(before.templates.getTemplates(), after.templates.getTemplates(), DiffableUtils.getStringKeySerializer()); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); } @@ -1059,7 +1084,7 @@ public Metadata apply(Metadata part) { builder.persistentSettings(persistentSettings); builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings)); builder.indices(indices.apply(part.indices)); - builder.templates(templates.apply(part.templates)); + builder.templates(templates.apply(part.templates.getTemplates())); builder.customs(customs.apply(part.customs)); return builder.build(); } @@ -1103,10 +1128,7 @@ public void writeTo(StreamOutput out) throws IOException { for (IndexMetadata indexMetadata : this) { indexMetadata.writeTo(out); } - out.writeVInt(templates.size()); - for (final IndexTemplateMetadata cursor : templates.values()) { - cursor.writeTo(out); - } + templates.writeTo(out); // filter out custom states not supported by the other node int numberOfCustoms = 0; for (final Custom cursor : customs.values()) { @@ -1170,7 +1192,7 @@ public Builder(Metadata metadata) { this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings; this.version = metadata.version; this.indices = new HashMap<>(metadata.indices); - this.templates = new HashMap<>(metadata.templates); + this.templates = new HashMap<>(metadata.templates.getTemplates()); this.customs = new HashMap<>(metadata.customs); this.previousMetadata = metadata; } @@ -1249,6 +1271,11 @@ public Builder templates(Map templates) { return this; } + public Builder templates(TemplatesMetadata templatesMetadata) { + this.templates.putAll(templatesMetadata.getTemplates()); + return this; + } + public Builder put(String name, ComponentTemplate componentTemplate) { Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name); Map existingTemplates = Optional.ofNullable( @@ -1738,11 +1765,7 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon builder.endObject(); } - builder.startObject("templates"); - for (final IndexTemplateMetadata cursor : metadata.templates().values()) { - IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params); - } - builder.endObject(); + metadata.templatesMetadata().toXContent(builder, params); if (context == XContentContext.API) { builder.startObject("indices"); @@ -1804,16 +1827,13 @@ public static Metadata fromXContent(XContentParser parser) throws IOException { } else if ("hashes_of_consistent_settings".equals(currentFieldName)) { builder.hashesOfConsistentSettings(parser.mapStrings()); } else if ("templates".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); - } + builder.templates(TemplatesMetadata.fromXContent(parser)); } else { try { - Custom custom = parser.namedObject(Custom.class, currentFieldName, null); + Custom custom = Custom.fromXContent(parser, currentFieldName); builder.putCustom(custom.getWriteableName(), custom); } catch (NamedObjectNotFoundException ex) { logger.warn("Skipping unknown custom object with type {}", currentFieldName); - parser.skipChildren(); } } } else if (token.isValue()) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java index e3689d046193c..9b52bdd1b16c5 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java @@ -202,6 +202,10 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO XContentParser.Token token; List repository = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.START_OBJECT) { + // move to next token if parsing the whole object + token = parser.nextToken(); + } if (token == XContentParser.Token.FIELD_NAME) { String name = parser.currentName(); if (parser.nextToken() != XContentParser.Token.START_OBJECT) { diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java new file mode 100644 index 0000000000000..ff4c7f62fd9c5 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment { + public static TemplatesMetadata EMPTY_METADATA = builder().build(); + private final Map templates; + + public TemplatesMetadata() { + this(Collections.emptyMap()); + } + + public TemplatesMetadata(Map templates) { + this.templates = Collections.unmodifiableMap(templates); + } + + public static Builder builder() { + return new Builder(); + } + + public Map getTemplates() { + return this.templates; + } + + public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException { + return Builder.fromXContent(parser); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Builder.toXContent(this, builder, params); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(templates.size()); + for (final IndexTemplateMetadata cursor : templates.values()) { + cursor.writeTo(out); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TemplatesMetadata that = (TemplatesMetadata) o; + + return Objects.equals(templates, that.templates); + } + + @Override + public int hashCode() { + return templates != null ? templates.hashCode() : 0; + } + + public static class Builder { + private final Map templates; + + public Builder() { + this.templates = new HashMap(); + } + + public Builder(Map templates) { + this.templates = templates; + } + + public Builder put(IndexTemplateMetadata.Builder template) { + return put(template.build()); + } + + public Builder put(IndexTemplateMetadata template) { + templates.put(template.name(), template); + return this; + } + + public Builder removeTemplate(String templateName) { + templates.remove(templateName); + return this; + } + + public Builder templates(Map templates) { + this.templates.putAll(templates); + return this; + } + + public TemplatesMetadata build() { + return new TemplatesMetadata(templates); + } + + public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException { +// builder.startObject("templates-metadata"); + for(IndexTemplateMetadata cursor : templates.getTemplates().values()) { + IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params); + } +// builder.endObject(); + } + + public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + + // we might get here after the templates-metadata element, or on a fresh parser + XContentParser.Token token = parser.currentToken(); + String currentFieldName = parser.currentName(); + if (currentFieldName == null) { + token = parser.nextToken(); + if (token == XContentParser.Token.START_OBJECT) { + // move to the field name + token = parser.nextToken(); + if (token == XContentParser.Token.FIELD_NAME) { + // move to the next object + token = parser.nextToken(); + } + } + currentFieldName = parser.currentName(); + } + if (currentFieldName != null && token == XContentParser.Token.START_OBJECT) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); + } + } + return builder.build(); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4725f40076ce2..a9f59fc8b1b8c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -16,6 +16,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -24,7 +25,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; /** * Manifest file which contains the details of the uploaded entity metadata @@ -35,6 +39,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. + public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -48,6 +53,10 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField UPLOADED_COORDINATOR_METADATA = new ParseField("uploaded_coordinator_metadata"); + private static final ParseField UPLOADED_SETTINGS_METADATA = new ParseField("uploaded_settings_metadata"); + private static final ParseField UPLOADED_TEMPLATES_METADATA = new ParseField("uploaded_templates_metadata"); + private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata"); private static long term(Object[] fields) { return (long) fields[0]; @@ -97,6 +106,23 @@ private static String globalMetadataFileName(Object[] fields) { return (String) fields[11]; } + private static UploadedMetadataAttribute coordinationMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[11]; + } + + private static UploadedMetadataAttribute settingsMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[12]; + } + + private static UploadedMetadataAttribute templatesMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[13]; + } + + private static Map customMetadata(Object[] fields) { + List customs = (List) fields[14]; + return customs.stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())); + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> new ClusterMetadataManifest( @@ -133,11 +159,34 @@ private static String globalMetadataFileName(Object[] fields) { ) ); - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V1; + private static final ConstructingObjectParser PARSER_V2 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> new ClusterMetadataManifest( + term(fields), + version(fields), + clusterUUID(fields), + stateUUID(fields), + opensearchVersion(fields), + nodeId(fields), + committed(fields), + codecVersion(fields), + null, + indices(fields), + previousClusterUUID(fields), + clusterUUIDCommitted(fields), + coordinationMetadata(fields), + settingsMetadata(fields), + templatesMetadata(fields), + customMetadata(fields) + ) + ); + + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); + declareParser(PARSER_V2, CODEC_V2); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -156,14 +205,24 @@ private static void declareParser(ConstructingObjectParser= CODEC_V1) { + if (codec_version == CODEC_V1) { parser.declareInt(ConstructingObjectParser.constructorArg(), CODEC_VERSION_FIELD); parser.declareString(ConstructingObjectParser.constructorArg(), GLOBAL_METADATA_FIELD); + } else if (codec_version == CODEC_V2) { + parser.declareInt(ConstructingObjectParser.constructorArg(), CODEC_VERSION_FIELD); + parser.declareNamedObject(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_COORDINATOR_METADATA); + parser.declareNamedObject(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_SETTINGS_METADATA); + parser.declareNamedObject(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_TEMPLATES_METADATA); + parser.declareNamedObjects(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_CUSTOM_METADATA); } } private final int codecVersion; private final String globalMetadataFileName; + private final UploadedMetadataAttribute uploadedCoordinationMetadata; + private final UploadedMetadataAttribute uploadedSettingsMetadata; + private final UploadedMetadataAttribute uploadedTemplatesMetadata; + private final Map uploadedCustomMetadataMap; private final List indices; private final long clusterTerm; private final long stateVersion; @@ -223,6 +282,26 @@ public String getGlobalMetadataFileName() { return globalMetadataFileName; } + public UploadedMetadataAttribute getCoordinationMetadata() { + return uploadedCoordinationMetadata; + } + + public UploadedMetadataAttribute getSettingsMetadata() { + return uploadedSettingsMetadata; + } + + public UploadedMetadataAttribute getTemplatesMetadata() { + return uploadedTemplatesMetadata; + } + + public Map getCustomMetadataMap() { + return uploadedCustomMetadataMap; + } + + public boolean hasMetadataAttributesFiles() { + return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null || uploadedTemplatesMetadata != null || uploadedCustomMetadataMap != null; + } + public ClusterMetadataManifest( long clusterTerm, long version, @@ -236,6 +315,44 @@ public ClusterMetadataManifest( List indices, String previousClusterUUID, boolean clusterUUIDCommitted + ) { + this( + clusterTerm, + version, + clusterUUID, + stateUUID, + opensearchVersion, + nodeId, + committed, + codecVersion, + globalMetadataFileName, + indices, + previousClusterUUID, + clusterUUIDCommitted, + null, + null, + null, + null + ); + } + + public ClusterMetadataManifest( + long clusterTerm, + long version, + String clusterUUID, + String stateUUID, + Version opensearchVersion, + String nodeId, + boolean committed, + int codecVersion, + String globalMetadataFileName, + List indices, + String previousClusterUUID, + boolean clusterUUIDCommitted, + UploadedMetadataAttribute uploadedCoordinationMetadata, + UploadedMetadataAttribute uploadedSettingsMetadata, + UploadedMetadataAttribute uploadedTemplatesMetadata, + Map uploadedCustomMetadataMap ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -249,6 +366,10 @@ public ClusterMetadataManifest( this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; + this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; + this.uploadedSettingsMetadata = uploadedSettingsMetadata; + this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + this.uploadedCustomMetadataMap = Collections.unmodifiableMap(uploadedCustomMetadataMap); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -262,12 +383,27 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_2_12_0)) { + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + this.codecVersion = in.readInt(); + this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in); + this.uploadedSettingsMetadata = new UploadedMetadataAttribute(in); + this.uploadedTemplatesMetadata = new UploadedMetadataAttribute(in); + this.uploadedCustomMetadataMap = Collections.unmodifiableMap(in.readMap(StreamInput::readString, UploadedMetadataAttribute::new)); + this.globalMetadataFileName = null; + } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.uploadedCoordinationMetadata = null; + this.uploadedSettingsMetadata = null; + this.uploadedTemplatesMetadata = null; + this.uploadedCustomMetadataMap = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; + this.uploadedCoordinationMetadata = null; + this.uploadedSettingsMetadata = null; + this.uploadedTemplatesMetadata = null; + this.uploadedCustomMetadataMap = null; } } @@ -297,7 +433,23 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endArray(); builder.field(PREVIOUS_CLUSTER_UUID.getPreferredName(), getPreviousClusterUUID()); builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted()); - if (onOrAfterCodecVersion(CODEC_V1)) { + if (onOrAfterCodecVersion(CODEC_V2)) { + builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); + builder.startObject(UPLOADED_COORDINATOR_METADATA.getPreferredName()); + getCoordinationMetadata().toXContent(builder, params); + builder.endObject(); + builder.startObject(UPLOADED_SETTINGS_METADATA.getPreferredName()); + getSettingsMetadata().toXContent(builder, params); + builder.endObject(); + builder.startObject(UPLOADED_TEMPLATES_METADATA.getPreferredName()); + getTemplatesMetadata().toXContent(builder, params); + builder.endObject(); + builder.startObject(UPLOADED_CUSTOM_METADATA.getPreferredName()); + for (UploadedMetadataAttribute attribute: getCustomMetadataMap().values()) { + attribute.toXContent(builder, params); + } + builder.endObject(); + } else if (onOrAfterCodecVersion(CODEC_V1)) { builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } @@ -316,7 +468,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(indices); out.writeString(previousClusterUUID); out.writeBoolean(clusterUUIDCommitted); - if (out.getVersion().onOrAfter(Version.V_2_12_0)) { + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + out.writeInt(codecVersion); + uploadedCoordinationMetadata.writeTo(out); + uploadedSettingsMetadata.writeTo(out); + uploadedTemplatesMetadata.writeTo(out); + out.writeMap(uploadedCustomMetadataMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); } @@ -613,4 +771,69 @@ public static UploadedIndexMetadata fromXContent(XContentParser parser) throws I return PARSER.parse(parser, null); } } + + public static class UploadedMetadataAttribute implements Writeable, ToXContentFragment { + private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename"); + + private static final ObjectParser.NamedObjectParser PARSER; + + static { + ConstructingObjectParser innerParser = new ConstructingObjectParser<>( + "uploaded_metadata_attribute", + true, + (Object[] parsedObject, String name) -> { + String uploadedFilename = (String) parsedObject[0]; + return new UploadedMetadataAttribute(name, uploadedFilename); + } + ); + innerParser.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD); + PARSER = ((p, c, name) -> innerParser.parse(p, name)); + } + + private final String attributeName; + private final String uploadedFilename; + + public UploadedMetadataAttribute(String attributeName, String uploadedFilename) { + this.attributeName = attributeName; + this.uploadedFilename = uploadedFilename; + } + + public UploadedMetadataAttribute(StreamInput in) throws IOException { + this.attributeName = in.readString(); + this.uploadedFilename = in.readString(); + } + + public String getAttributeName() { + return attributeName; + } + + public String getUploadedFilename() { + return uploadedFilename; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(attributeName); + out.writeString(uploadedFilename); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject(getAttributeName()) + .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilename()) + .endObject(); + } + + public static UploadedMetadataAttribute fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null, parser.currentName()); + } + + @Override + public String toString() { + return "UploadedMetadataAttribute{" + + "attributeName='" + attributeName + '\'' + + ", uploadedFilename='" + uploadedFilename + '\'' + + '}'; + } + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c892b475d71da..13a1a565886fa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -14,8 +14,10 @@ import org.opensearch.Version; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -30,6 +32,7 @@ import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.node.Node; @@ -39,6 +42,7 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; +import reactor.util.annotation.NonNull; import java.io.Closeable; import java.io.IOException; @@ -122,6 +126,32 @@ public class RemoteClusterStateService implements Closeable { Metadata::fromXContent ); + public static final ChecksumBlobStoreFormat COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "coordination", + METADATA_NAME_FORMAT, + CoordinationMetadata::fromXContent + ); + + public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "settings", + METADATA_NAME_FORMAT, + Settings::fromXContent + ); + + public static final ChecksumBlobStoreFormat TEMPLATE_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "template", + METADATA_NAME_FORMAT, + TemplatesMetadata::fromXContent + ); + + public static final ChecksumBlobStoreFormat CUSTOM_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "custom", + METADATA_NAME_FORMAT, + Metadata.Custom::fromXContent + ); + + private static final Map metadataComponentBlobStoreMap = new HashMap<>(); + /** * Manifest format compatible with older codec v0, where codec version was missing. */ @@ -153,6 +183,10 @@ public class RemoteClusterStateService implements Closeable { public static final String MANIFEST_PATH_TOKEN = "manifest"; public static final String MANIFEST_FILE_PREFIX = "manifest"; public static final String METADATA_FILE_PREFIX = "metadata"; + public static final String COORDINATION_METADATA = "coordination"; + public static final String SETTING_METADATA = "settings"; + public static final String TEMPLATE_METADATA = "template"; + public static final String CUSTOM_METADATA = "custom"; public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion private final String nodeId; @@ -171,7 +205,7 @@ public class RemoteClusterStateService implements Closeable { private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; // ToXContent Params with gateway mode. @@ -206,6 +240,11 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); + + metadataComponentBlobStoreMap.put("coordination", COORDINATION_METADATA_FORMAT); + metadataComponentBlobStoreMap.put("settings", SETTINGS_METADATA_FORMAT); + metadataComponentBlobStoreMap.put("templates", TEMPLATE_METADATA_FORMAT); + metadataComponentBlobStoreMap.put("customs", CUSTOM_METADATA_FORMAT); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -230,8 +269,11 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri } // TODO: we can upload global metadata and index metadata in parallel. [issue: #10645] - // Write globalMetadata - String globalMetadataFile = writeGlobalMetadata(clusterState); + // Write globalMetadata fragments + UploadedMetadataAttribute uploadedCoordinationMetadata = writeCoordinationMetadata(clusterState); + UploadedMetadataAttribute uploadedSettingsMetadata = writeSettingsMetadata(clusterState); + UploadedMetadataAttribute uploadedTemplateMetadata = writeTemplateMetadata(clusterState); + Map uploadedCustomMetadataMap = writeCustomMetadataInParallel(clusterState); // any validations before/after upload ? final List allUploadedIndexMetadata = writeIndexMetadataParallel( @@ -242,7 +284,10 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri clusterState, allUploadedIndexMetadata, previousClusterUUID, - globalMetadataFile, + uploadedCoordinationMetadata, + uploadedSettingsMetadata, + uploadedTemplateMetadata, + uploadedCustomMetadataMap, false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); @@ -290,14 +335,23 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousClusterState.metadata(), clusterState.metadata() ) == false; - String globalMetadataFile; - // For migration case from codec V0 to V1, we have added null check on global metadata file, + UploadedMetadataAttribute uploadedCoordinationMetadata; + UploadedMetadataAttribute uploadedSettingsMetadata; + UploadedMetadataAttribute uploadedTemplateMetadata; + Map uploadedCustomMetadataMap; + // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, // If file is empty and codec is 1 then write global metadata. - if (updateGlobalMetadata || previousManifest.getGlobalMetadataFileName() == null) { - globalMetadataFile = writeGlobalMetadata(clusterState); + if (updateGlobalMetadata || !previousManifest.hasMetadataAttributesFiles()) { + uploadedCoordinationMetadata = writeCoordinationMetadata(clusterState); + uploadedSettingsMetadata = writeSettingsMetadata(clusterState); + uploadedTemplateMetadata = writeTemplateMetadata(clusterState); + uploadedCustomMetadataMap = writeCustomMetadataInParallel(clusterState); } else { logger.debug("Global metadata has not updated in cluster state, skipping upload of it"); - globalMetadataFile = previousManifest.getGlobalMetadataFileName(); + uploadedCoordinationMetadata = previousManifest.getCoordinationMetadata(); + uploadedSettingsMetadata = previousManifest.getSettingsMetadata(); + uploadedTemplateMetadata = previousManifest.getTemplatesMetadata(); + uploadedCustomMetadataMap = previousManifest.getCustomMetadataMap(); } // Write Index Metadata @@ -343,7 +397,10 @@ public ClusterMetadataManifest writeIncrementalMetadata( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), previousManifest.getPreviousClusterUUID(), - globalMetadataFile, + uploadedCoordinationMetadata, + uploadedSettingsMetadata, + uploadedTemplateMetadata, + uploadedCustomMetadataMap, false ); deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS); @@ -432,6 +489,194 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException return result.get(); } + private UploadedMetadataAttribute writeCoordinationMetadata(ClusterState clusterState) throws IOException { + AtomicReference result = new AtomicReference<>(); + AtomicReference exceptionReference = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + + LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( + (UploadedMetadataAttribute uploadedMetadataAttribute) -> { + logger.trace(String.format(Locale.ROOT, "CoordinationMetadata uploaded successfully.")); + result.set(uploadedMetadataAttribute); + }, ex -> { + logger.error( + () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), + ex + ); + exceptionReference.set(ex); + }), + latch); + + writeMetadataComponentAsync(clusterState, COORDINATION_METADATA, COORDINATION_METADATA_FORMAT, clusterState.metadata().coordinationMetadata(), completionListener); + + try { + if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + RemoteStateTransferException ex = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of coordination metadata to complete") + ); + throw ex; + } + } catch (InterruptedException ex) { + RemoteStateTransferException exception = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of coordination metadata to complete - %s"), + ex + ); + Thread.currentThread().interrupt(); + throw exception; + } + if (exceptionReference.get() != null) { + throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); + } + return result.get(); + } + + private UploadedMetadataAttribute writeSettingsMetadata(ClusterState clusterState) throws IOException { + AtomicReference result = new AtomicReference<>(); + AtomicReference exceptionReference = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + + LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( + (UploadedMetadataAttribute uploadedMetadataAttribute) -> { + logger.trace(String.format(Locale.ROOT, "Settings Metadata uploaded successfully.")); + result.set(uploadedMetadataAttribute); + }, ex -> { + logger.error( + () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), + ex + ); + exceptionReference.set(ex); + }), + latch); + + writeMetadataComponentAsync(clusterState, SETTING_METADATA, SETTINGS_METADATA_FORMAT, clusterState.metadata().persistentSettings(), completionListener); + + try { + if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + RemoteStateTransferException ex = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of settings metadata to complete") + ); + throw ex; + } + } catch (InterruptedException ex) { + RemoteStateTransferException exception = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of settings metadata to complete - %s"), + ex + ); + Thread.currentThread().interrupt(); + throw exception; + } + if (exceptionReference.get() != null) { + throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); + } + return result.get(); + } + + private UploadedMetadataAttribute writeTemplateMetadata(ClusterState clusterState) throws IOException { + AtomicReference result = new AtomicReference<>(); + AtomicReference exceptionReference = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + + LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( + (UploadedMetadataAttribute uploadedMetadataAttribute) -> { + logger.trace(String.format(Locale.ROOT, "Templates Metadata uploaded successfully.")); + result.set(uploadedMetadataAttribute); + }, ex -> { + logger.error( + () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), + ex + ); + exceptionReference.set(ex); + }), + latch); + + writeMetadataComponentAsync(clusterState, TEMPLATE_METADATA, TEMPLATE_METADATA_FORMAT, clusterState.metadata().templatesMetadata(), completionListener); + + try { + if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + RemoteStateTransferException ex = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of templates metadata to complete") + ); + throw ex; + } + } catch (InterruptedException ex) { + RemoteStateTransferException exception = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of templates metadata to complete - %s"), + ex + ); + Thread.currentThread().interrupt(); + throw exception; + } + if (exceptionReference.get() != null) { + throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); + } + return result.get(); + } + + private Map writeCustomMetadataInParallel(ClusterState clusterState) throws IOException { + Map result = new HashMap<>(); + List exceptionReference = new ArrayList<>(); + + Map customToUpload = clusterState.metadata().customs(); + + final CountDownLatch latch = new CountDownLatch(customToUpload.size()); + + LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( + (UploadedMetadataAttribute uploadedMetadataAttribute) -> { + logger.trace(String.format(Locale.ROOT, "Custom Metadata uploaded successfully.")); + result.put(uploadedMetadataAttribute.getAttributeName(), uploadedMetadataAttribute); + }, ex -> { + logger.error( + () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), + ex + ); + exceptionReference.add(ex); + }), + latch + ); + + for (Map.Entry custom : customToUpload.entrySet()) { + writeMetadataComponentAsync(clusterState, custom.getKey(), CUSTOM_METADATA_FORMAT, custom.getValue(), completionListener); + } + + try { + if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + RemoteStateTransferException ex = new RemoteStateTransferException( + String.format( + Locale.ROOT, + "Timed out waiting for transfer of custom metadata to complete - %s", + String.join(", ", customToUpload.keySet()) + ) + ); + exceptionReference.forEach(ex::addSuppressed); + throw ex; + } + } catch (InterruptedException ex) { + exceptionReference.forEach(ex::addSuppressed); + RemoteStateTransferException exception = new RemoteStateTransferException( + String.format( + Locale.ROOT, + "Timed out waiting for transfer of custom metadata to complete - %s", + String.join(", ", customToUpload.keySet()) + ), + ex + ); + Thread.currentThread().interrupt(); + throw exception; + } + if (!exceptionReference.isEmpty()) { + RemoteStateTransferException exception = new RemoteStateTransferException( + String.format( + Locale.ROOT, + "Exception during transfer of custom metadata to complete - %s", + String.join(", ", customToUpload.keySet()) + ) + ); + exceptionReference.forEach(exception::addSuppressed); + throw exception; + } + return result; + } + /** * Uploads provided IndexMetadata's to remote store in parallel. The call is blocking so the method waits for upload to finish and then return. * @@ -545,6 +790,42 @@ private void writeIndexMetadataAsync( ); } + /** + * Allows async upload of Metadata components to remote + */ + + private void writeMetadataComponentAsync( + ClusterState clusterState, + String component, + ChecksumBlobStoreFormat componentMetadataBlobStore, + ToXContent componentMetadata, + LatchedActionListener latchedActionListener + ) throws IOException { + final BlobContainer globalMetadataContainer = globalMetadataContainer( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID() + ); + final String componentMetadataFilename = metadataAttributeFileName(component, clusterState.metadata().version()); + ActionListener completionListener = ActionListener.wrap( + resp -> latchedActionListener.onResponse( + new UploadedMetadataAttribute( + component, + componentMetadataFilename + ) + ), + ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, ex)) + ); + + componentMetadataBlobStore.writeAsyncWithUrgentPriority( + componentMetadata, + globalMetadataContainer, + componentMetadataFilename, + blobStoreRepository.getCompressor(), + completionListener, + FORMAT_PARAMS + ); + } + @Nullable public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) throws IOException { @@ -558,7 +839,10 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat clusterState, previousManifest.getIndices(), previousManifest.getPreviousClusterUUID(), - previousManifest.getGlobalMetadataFileName(), + previousManifest.getCoordinationMetadata(), + previousManifest.getSettingsMetadata(), + previousManifest.getTemplatesMetadata(), + previousManifest.getCustomMetadataMap(), true ); deleteStaleClusterUUIDs(clusterState, committedManifest); @@ -587,7 +871,10 @@ private ClusterMetadataManifest uploadManifest( ClusterState clusterState, List uploadedIndexMetadata, String previousClusterUUID, - String globalClusterMetadataFileName, + UploadedMetadataAttribute uploadedCoordinationMetadata, + UploadedMetadataAttribute uploadedSettingsMetadata, + UploadedMetadataAttribute uploadedTemplateMetadata, + Map uploadedCustomMetadataMap, boolean committed ) throws IOException { synchronized (this) { @@ -601,10 +888,14 @@ private ClusterMetadataManifest uploadManifest( nodeId, committed, MANIFEST_CURRENT_CODEC_VERSION, - globalClusterMetadataFileName, + null, uploadedIndexMetadata, previousClusterUUID, - clusterState.metadata().clusterUUIDCommitted() + clusterState.metadata().clusterUUIDCommitted(), + uploadedCoordinationMetadata, + uploadedSettingsMetadata, + uploadedTemplateMetadata, + uploadedCustomMetadataMap ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; @@ -765,6 +1056,17 @@ private static String globalMetadataFileName(Metadata metadata) { ); } + private static String metadataAttributeFileName(String componentPrefix, Long metadataVersion) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/settings______ + return String.join( + DELIMITER, + componentPrefix, + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + } + private BlobPath getManifestFolderPath(String clusterName, String clusterUUID) { return getCusterMetadataBasePath(clusterName, clusterUUID).add(MANIFEST_PATH_TOKEN); } @@ -832,6 +1134,7 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID String.format(Locale.ROOT, "Latest cluster metadata manifest is not present for the provided clusterUUID: %s", clusterUUID) ); } + // Fetch Global Metadata Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get()); @@ -858,6 +1161,22 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust splitPath[splitPath.length - 1], blobStoreRepository.getNamedXContentRegistry() ); + } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(clusterName, clusterUUID, + clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); + Settings settingsMetadata = getSettingsMetadata(clusterName, clusterUUID, + clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); + TemplatesMetadata templatesMetadata = getTemplatesMetadata(clusterName, clusterUUID, + clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); + Metadata.Builder builder = new Metadata.Builder(); + builder.coordinationMetadata(coordinationMetadata); + builder.persistentSettings(settingsMetadata); + builder.templates(templatesMetadata); + clusterMetadataManifest.getCustomMetadataMap().forEach( + (key, value) -> + builder.putCustom(key, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key)) + ); + return builder.build(); } else { return Metadata.EMPTY_METADATA; } @@ -869,6 +1188,91 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust } } + private CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { + try { + // Fetch Coordintaion metadata + if (coordinationMetadataFileName != null) { + String[] splitPath = coordinationMetadataFileName.split("/"); + return COORDINATION_METADATA_FORMAT.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else { + return CoordinationMetadata.EMPTY_METADATA; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Coordination Metadata - %s", coordinationMetadataFileName), + e + ); + } + } + + private Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { + try { + // Fetch Settings metadata + if (settingsMetadataFileName != null) { + String[] splitPath = settingsMetadataFileName.split("/"); + return SETTINGS_METADATA_FORMAT.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else { + return Settings.EMPTY; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Settings Metadata - %s", settingsMetadataFileName), + e + ); + } + } + + private TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { + try { + // Fetch Templates metadata + if (templatesMetadataFileName != null) { + String[] splitPath = templatesMetadataFileName.split("/"); + return TEMPLATE_METADATA_FORMAT.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else { + return TemplatesMetadata.EMPTY_METADATA; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", templatesMetadataFileName), + e + ); + } + } + + private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, @NonNull String customMetadataFileName, String custom) { + try { + // Fetch Custom metadata + String[] splitPath = customMetadataFileName.split("/"); + ChecksumBlobStoreFormat customChecksumBlobStoreFormat = new ChecksumBlobStoreFormat<>( + "custom", + METADATA_NAME_FORMAT, + (parser -> Metadata.Custom.fromXContent(parser, custom)) + ); + return customChecksumBlobStoreFormat.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), + e + ); + } + } + /** * Fetch latest ClusterMetadataManifest from remote state store * From 5d6a0ad25b77c2ba69e8480559b428893df671b6 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 9 Feb 2024 01:43:30 +0530 Subject: [PATCH 002/133] Upload all the metadata attributes in parallel Signed-off-by: Shivansh Arora --- .../cluster/metadata/IndexMetadata.java | 4 + .../opensearch/cluster/metadata/Metadata.java | 33 +- .../cluster/metadata/TemplatesMetadata.java | 11 +- .../remote/ClusterMetadataManifest.java | 140 +++- .../remote/RemoteClusterStateService.java | 688 ++++++++---------- .../remote/ClusterMetadataManifestTests.java | 45 +- .../RemoteClusterStateServiceTests.java | 600 +++++++++++---- 7 files changed, 964 insertions(+), 557 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 03784df509ed6..5adb07bd91572 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -752,6 +752,10 @@ public String getIndexUUID() { return index.getUUID(); } + public String getIndexName() { + return index.getName(); + } + /** * Test whether the current index UUID is the same as the given one. Returns true if either are _na_ */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 464a6e89effbd..0141a74d01d0c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -948,7 +948,7 @@ public Iterator iterator() { } public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) { - if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) { + if (!isCoordinationMetadataEqual(metadata1, metadata2)) { return false; } if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) { @@ -967,13 +967,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2 * Compares Metadata entities persisted in Remote Store. */ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) { - if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) { + if (!isSettingsMetadataEqual(metadata1, metadata2)) { return false; } - if (!metadata1.templates.equals(metadata2.templates)) { + if (!isTemplatesMetadataEqual(metadata1, metadata2)) { return false; } // Check if any persistent metadata needs to be saved + return isCustomMetadataEqual(metadata1, metadata2); + } + + public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata); + } + + public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.persistentSettings.equals(metadata2.persistentSettings); + } + + public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.templates.equals(metadata2.templates); + } + + public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) { int customCount1 = 0; for (Map.Entry cursor : metadata1.customs.entrySet()) { if (cursor.getValue().context().contains(XContentContext.GATEWAY)) { @@ -987,8 +1003,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada customCount2++; } } - if (customCount1 != customCount2) return false; - return true; + return customCount1 == customCount2; } @Override @@ -1037,7 +1052,11 @@ private static class MetadataDiff implements Diff { persistentSettings = after.persistentSettings; hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings); indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer()); - templates = DiffableUtils.diff(before.templates.getTemplates(), after.templates.getTemplates(), DiffableUtils.getStringKeySerializer()); + templates = DiffableUtils.diff( + before.templates.getTemplates(), + after.templates.getTemplates(), + DiffableUtils.getStringKeySerializer() + ); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); } @@ -1765,7 +1784,9 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon builder.endObject(); } + builder.startObject("templates"); metadata.templatesMetadata().toXContent(builder, params); + builder.endObject(); if (context == XContentContext.API) { builder.startObject("indices"); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index ff4c7f62fd9c5..179d5aa6fbc52 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -9,9 +9,6 @@ package org.opensearch.cluster.metadata; import org.opensearch.cluster.AbstractDiffable; -import org.opensearch.cluster.coordination.CoordinationMetadata; -import org.opensearch.common.annotation.PublicApi; -import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -66,7 +63,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - TemplatesMetadata that = (TemplatesMetadata) o; + TemplatesMetadata that = (TemplatesMetadata) o; return Objects.equals(templates, that.templates); } @@ -111,11 +108,11 @@ public TemplatesMetadata build() { } public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException { -// builder.startObject("templates-metadata"); - for(IndexTemplateMetadata cursor : templates.getTemplates().values()) { + // builder.startObject("templates-metadata"); + for (IndexTemplateMetadata cursor : templates.getTemplates().values()) { IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params); } -// builder.endObject(); + // builder.endObject(); } public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index a9f59fc8b1b8c..a3a84924e4563 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -210,10 +211,26 @@ private static void declareParser(ConstructingObjectParser getCustomMetadataMap() { } public boolean hasMetadataAttributesFiles() { - return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null || uploadedTemplatesMetadata != null || uploadedCustomMetadataMap != null; + return uploadedCoordinationMetadata != null + || uploadedSettingsMetadata != null + || uploadedTemplatesMetadata != null + || !uploadedCustomMetadataMap.isEmpty(); } public ClusterMetadataManifest( @@ -369,7 +389,9 @@ public ClusterMetadataManifest( this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; this.uploadedSettingsMetadata = uploadedSettingsMetadata; this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; - this.uploadedCustomMetadataMap = Collections.unmodifiableMap(uploadedCustomMetadataMap); + this.uploadedCustomMetadataMap = Collections.unmodifiableMap( + uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>() + ); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -388,7 +410,9 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in); this.uploadedSettingsMetadata = new UploadedMetadataAttribute(in); this.uploadedTemplatesMetadata = new UploadedMetadataAttribute(in); - this.uploadedCustomMetadataMap = Collections.unmodifiableMap(in.readMap(StreamInput::readString, UploadedMetadataAttribute::new)); + this.uploadedCustomMetadataMap = Collections.unmodifiableMap( + in.readMap(StreamInput::readString, UploadedMetadataAttribute::new) + ); this.globalMetadataFileName = null; } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); @@ -435,17 +459,23 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted()); if (onOrAfterCodecVersion(CODEC_V2)) { builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); - builder.startObject(UPLOADED_COORDINATOR_METADATA.getPreferredName()); - getCoordinationMetadata().toXContent(builder, params); - builder.endObject(); - builder.startObject(UPLOADED_SETTINGS_METADATA.getPreferredName()); - getSettingsMetadata().toXContent(builder, params); - builder.endObject(); - builder.startObject(UPLOADED_TEMPLATES_METADATA.getPreferredName()); - getTemplatesMetadata().toXContent(builder, params); - builder.endObject(); + if (getCoordinationMetadata() != null) { + builder.startObject(UPLOADED_COORDINATOR_METADATA.getPreferredName()); + getCoordinationMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getSettingsMetadata() != null) { + builder.startObject(UPLOADED_SETTINGS_METADATA.getPreferredName()); + getSettingsMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getTemplatesMetadata() != null) { + builder.startObject(UPLOADED_TEMPLATES_METADATA.getPreferredName()); + getTemplatesMetadata().toXContent(builder, params); + builder.endObject(); + } builder.startObject(UPLOADED_CUSTOM_METADATA.getPreferredName()); - for (UploadedMetadataAttribute attribute: getCustomMetadataMap().values()) { + for (UploadedMetadataAttribute attribute : getCustomMetadataMap().values()) { attribute.toXContent(builder, params); } builder.endObject(); @@ -534,6 +564,10 @@ public static ClusterMetadataManifest fromXContentV0(XContentParser parser) thro return PARSER_V0.parse(parser, null); } + public static ClusterMetadataManifest fromXContentV1(XContentParser parser) throws IOException { + return PARSER_V1.parse(parser, null); + } + public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException { return CURRENT_PARSER.parse(parser, null); } @@ -546,6 +580,10 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws public static class Builder { private String globalMetadataFileName; + private UploadedMetadataAttribute coordinationMetadata; + private UploadedMetadataAttribute settingsMetadata; + private UploadedMetadataAttribute templatesMetadata; + private Map customMetadataMap; private int codecVersion; private List indices; private long clusterTerm; @@ -573,6 +611,31 @@ public Builder globalMetadataFileName(String globalMetadataFileName) { return this; } + public Builder coordinationMetadata(UploadedMetadataAttribute coordinationMetadata) { + this.coordinationMetadata = coordinationMetadata; + return this; + } + + public Builder settingMetadata(UploadedMetadataAttribute settingsMetadata) { + this.settingsMetadata = settingsMetadata; + return this; + } + + public Builder templatesMetadata(UploadedMetadataAttribute templatesMetadata) { + this.templatesMetadata = templatesMetadata; + return this; + } + + public Builder customMetadataMap(Map customMetadataMap) { + this.customMetadataMap = customMetadataMap; + return this; + } + + public Builder put(String custom, UploadedMetadataAttribute customMetadata) { + this.customMetadataMap.put(custom, customMetadata); + return this; + } + public Builder clusterTerm(long clusterTerm) { this.clusterTerm = clusterTerm; return this; @@ -624,6 +687,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { public Builder() { indices = new ArrayList<>(); + customMetadataMap = new HashMap<>(); } public Builder(ClusterMetadataManifest manifest) { @@ -635,6 +699,10 @@ public Builder(ClusterMetadataManifest manifest) { this.nodeId = manifest.nodeId; this.committed = manifest.committed; this.globalMetadataFileName = manifest.globalMetadataFileName; + this.coordinationMetadata = manifest.uploadedCoordinationMetadata; + this.settingsMetadata = manifest.uploadedSettingsMetadata; + this.templatesMetadata = manifest.uploadedTemplatesMetadata; + this.customMetadataMap = manifest.uploadedCustomMetadataMap; this.codecVersion = manifest.codecVersion; this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; @@ -654,18 +722,28 @@ public ClusterMetadataManifest build() { globalMetadataFileName, indices, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + coordinationMetadata, + settingsMetadata, + templatesMetadata, + customMetadataMap ); } } + public static interface UploadedMetadata { + String getComponent(); + + String getUploadedFilename(); + } + /** * Metadata for uploaded index metadata * * @opensearch.internal */ - public static class UploadedIndexMetadata implements Writeable, ToXContentFragment { + public static class UploadedIndexMetadata implements UploadedMetadata, Writeable, ToXContentFragment { private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name"); private static final ParseField INDEX_UUID_FIELD = new ParseField("index_uuid"); @@ -714,6 +792,11 @@ public String getUploadedFilePath() { return uploadedFilename; } + @Override + public String getComponent() { + return getIndexName(); + } + public String getUploadedFilename() { String[] splitPath = uploadedFilename.split("/"); return splitPath[splitPath.length - 1]; @@ -772,7 +855,7 @@ public static UploadedIndexMetadata fromXContent(XContentParser parser) throws I } } - public static class UploadedMetadataAttribute implements Writeable, ToXContentFragment { + public static class UploadedMetadataAttribute implements UploadedMetadata, Writeable, ToXContentFragment { private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename"); private static final ObjectParser.NamedObjectParser PARSER; @@ -807,6 +890,11 @@ public String getAttributeName() { return attributeName; } + @Override + public String getComponent() { + return getAttributeName(); + } + public String getUploadedFilename() { return uploadedFilename; } @@ -830,10 +918,14 @@ public static UploadedMetadataAttribute fromXContent(XContentParser parser) thro @Override public String toString() { - return "UploadedMetadataAttribute{" + - "attributeName='" + attributeName + '\'' + - ", uploadedFilename='" + uploadedFilename + '\'' + - '}'; + return "UploadedMetadataAttribute{" + + "attributeName='" + + attributeName + + '\'' + + ", uploadedFilename='" + + uploadedFilename + + '\'' + + '}'; } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 13a1a565886fa..718197618ccf1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,6 +18,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -29,7 +30,6 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; @@ -42,7 +42,6 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; -import reactor.util.annotation.NonNull; import java.io.Closeable; import java.io.IOException; @@ -67,6 +66,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import reactor.util.annotation.NonNull; + import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; @@ -138,8 +139,8 @@ public class RemoteClusterStateService implements Closeable { Settings::fromXContent ); - public static final ChecksumBlobStoreFormat TEMPLATE_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "template", + public static final ChecksumBlobStoreFormat TEMPLATES_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "templates", METADATA_NAME_FORMAT, TemplatesMetadata::fromXContent ); @@ -150,8 +151,6 @@ public class RemoteClusterStateService implements Closeable { Metadata.Custom::fromXContent ); - private static final Map metadataComponentBlobStoreMap = new HashMap<>(); - /** * Manifest format compatible with older codec v0, where codec version was missing. */ @@ -159,7 +158,13 @@ public class RemoteClusterStateService implements Closeable { new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); /** - * Manifest format compatible with codec v1, where we introduced codec versions/global metadata. + * Manifest format compatible with older codec v1, where global metadata was missing. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); + + /** + * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( "cluster-metadata-manifest", @@ -185,7 +190,7 @@ public class RemoteClusterStateService implements Closeable { public static final String METADATA_FILE_PREFIX = "metadata"; public static final String COORDINATION_METADATA = "coordination"; public static final String SETTING_METADATA = "settings"; - public static final String TEMPLATE_METADATA = "template"; + public static final String TEMPLATES_METADATA = "templates"; public static final String CUSTOM_METADATA = "custom"; public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion @@ -240,11 +245,6 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); - - metadataComponentBlobStoreMap.put("coordination", COORDINATION_METADATA_FORMAT); - metadataComponentBlobStoreMap.put("settings", SETTINGS_METADATA_FORMAT); - metadataComponentBlobStoreMap.put("templates", TEMPLATE_METADATA_FORMAT); - metadataComponentBlobStoreMap.put("customs", CUSTOM_METADATA_FORMAT); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -268,26 +268,22 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri return null; } - // TODO: we can upload global metadata and index metadata in parallel. [issue: #10645] - // Write globalMetadata fragments - UploadedMetadataAttribute uploadedCoordinationMetadata = writeCoordinationMetadata(clusterState); - UploadedMetadataAttribute uploadedSettingsMetadata = writeSettingsMetadata(clusterState); - UploadedMetadataAttribute uploadedTemplateMetadata = writeTemplateMetadata(clusterState); - Map uploadedCustomMetadataMap = writeCustomMetadataInParallel(clusterState); - - // any validations before/after upload ? - final List allUploadedIndexMetadata = writeIndexMetadataParallel( + UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel( clusterState, - new ArrayList<>(clusterState.metadata().indices().values()) + new ArrayList<>(clusterState.metadata().indices().values()), + clusterState.metadata().customs(), + true, + true, + true ); final ClusterMetadataManifest manifest = uploadManifest( clusterState, - allUploadedIndexMetadata, + uploadedMetadataResults.uploadedIndexMetadata, previousClusterUUID, - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplateMetadata, - uploadedCustomMetadataMap, + uploadedMetadataResults.uploadedCoordinationMetadata, + uploadedMetadataResults.uploadedSettingsMetadata, + uploadedMetadataResults.uploadedTemplatesMetadata, + uploadedMetadataResults.uploadedCustomMetadataMap, false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); @@ -298,13 +294,13 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " + "wrote full state with [{}] indices", durationMillis, slowWriteLoggingThreshold, - allUploadedIndexMetadata.size() + uploadedMetadataResults.uploadedIndexMetadata.size() ); } else { logger.info( "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices and global metadata", durationMillis, - allUploadedIndexMetadata.size() + uploadedMetadataResults.uploadedIndexMetadata.size() ); } return manifest; @@ -335,23 +331,24 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousClusterState.metadata(), clusterState.metadata() ) == false; - UploadedMetadataAttribute uploadedCoordinationMetadata; - UploadedMetadataAttribute uploadedSettingsMetadata; - UploadedMetadataAttribute uploadedTemplateMetadata; - Map uploadedCustomMetadataMap; - // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, - // If file is empty and codec is 1 then write global metadata. - if (updateGlobalMetadata || !previousManifest.hasMetadataAttributesFiles()) { - uploadedCoordinationMetadata = writeCoordinationMetadata(clusterState); - uploadedSettingsMetadata = writeSettingsMetadata(clusterState); - uploadedTemplateMetadata = writeTemplateMetadata(clusterState); - uploadedCustomMetadataMap = writeCustomMetadataInParallel(clusterState); - } else { - logger.debug("Global metadata has not updated in cluster state, skipping upload of it"); - uploadedCoordinationMetadata = previousManifest.getCoordinationMetadata(); - uploadedSettingsMetadata = previousManifest.getSettingsMetadata(); - uploadedTemplateMetadata = previousManifest.getTemplatesMetadata(); - uploadedCustomMetadataMap = previousManifest.getCustomMetadataMap(); + + final boolean updateCoordinationMetadata = Metadata.isCoordinationMetadataEqual( + previousClusterState.metadata(), + clusterState.metadata() + ) == false; + final boolean updateSettingsMetadata = Metadata.isSettingsMetadataEqual( + previousClusterState.metadata(), + clusterState.metadata() + ) == false; + final boolean updateTemplatesMetadata = Metadata.isTemplatesMetadataEqual( + previousClusterState.metadata(), + clusterState.metadata() + ) == false; + final Map previousStateCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); + final Map customsToUpload = getUpdatedCustoms(clusterState, previousClusterState); + final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); + for (final String custom : clusterState.metadata().customs().keySet()) { + previousStateCustomMap.remove(custom); } // Write Index Metadata @@ -384,12 +381,28 @@ public ClusterMetadataManifest writeIncrementalMetadata( } previousStateIndexMetadataVersionByName.remove(indexMetadata.getIndex().getName()); } + UploadedMetadataResults uploadedMetadataResults; + boolean firstUpload = !previousManifest.hasMetadataAttributesFiles(); + // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, + // If file is empty and codec is 1 then write global metadata. + if (firstUpload) { + uploadedMetadataResults = writeMetadataInParallel(clusterState, toUpload, clusterState.metadata().customs(), true, true, true); + } else { + uploadedMetadataResults = writeMetadataInParallel( + clusterState, + toUpload, + customsToUpload, + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata + ); + } - List uploadedIndexMetadataList = writeIndexMetadataParallel(clusterState, toUpload); - uploadedIndexMetadataList.forEach( + uploadedMetadataResults.uploadedIndexMetadata.forEach( uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata) ); - + allUploadedCustomMap.putAll(uploadedMetadataResults.uploadedCustomMetadataMap); + previousStateCustomMap.keySet().forEach(allUploadedCustomMap::remove); for (String removedIndexName : previousStateIndexMetadataVersionByName.keySet()) { allUploadedIndexMetadata.remove(removedIndexName); } @@ -397,10 +410,16 @@ public ClusterMetadataManifest writeIncrementalMetadata( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), previousManifest.getPreviousClusterUUID(), - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplateMetadata, - uploadedCustomMetadataMap, + firstUpload || updateCoordinationMetadata + ? uploadedMetadataResults.uploadedCoordinationMetadata + : previousManifest.getCoordinationMetadata(), + firstUpload || updateSettingsMetadata + ? uploadedMetadataResults.uploadedSettingsMetadata + : previousManifest.getSettingsMetadata(), + firstUpload || updateTemplatesMetadata + ? uploadedMetadataResults.uploadedTemplatesMetadata + : previousManifest.getTemplatesMetadata(), + firstUpload || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), false ); deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS); @@ -411,344 +430,170 @@ public ClusterMetadataManifest writeIncrementalMetadata( if (durationMillis >= slowWriteLoggingThreshold.getMillis()) { logger.warn( "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " - + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]", + + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " + + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", durationMillis, slowWriteLoggingThreshold, numIndicesUpdated, numIndicesUnchanged, - updateGlobalMetadata + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata, + customsToUpload.size() ); } else { logger.info( "writing cluster state for version [{}] took [{}ms]; " - + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]", + + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " + + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", manifest.getStateVersion(), durationMillis, numIndicesUpdated, numIndicesUnchanged, - updateGlobalMetadata + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata, + customsToUpload.size() ); } return manifest; } - /** - * Uploads provided ClusterState's global Metadata to remote store in parallel. - * The call is blocking so the method waits for upload to finish and then return. - * - * @param clusterState current ClusterState - * @return String file name where globalMetadata file is stored. - */ - private String writeGlobalMetadata(ClusterState clusterState) throws IOException { - - AtomicReference result = new AtomicReference(); - AtomicReference exceptionReference = new AtomicReference(); - - final BlobContainer globalMetadataContainer = globalMetadataContainer( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() - ); - final String globalMetadataFilename = globalMetadataFileName(clusterState.metadata()); - - // latch to wait until upload is not finished - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> { - logger.trace(String.format(Locale.ROOT, "GlobalMetadata uploaded successfully.")); - result.set(globalMetadataContainer.path().buildAsString() + globalMetadataFilename); - }, ex -> { exceptionReference.set(ex); }), latch); - - GLOBAL_METADATA_FORMAT.writeAsyncWithUrgentPriority( - clusterState.metadata(), - globalMetadataContainer, - globalMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - // TODO: We should add metrics where transfer is timing out. [Issue: #10687] - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of global metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); - } - return result.get(); - } - - private UploadedMetadataAttribute writeCoordinationMetadata(ClusterState clusterState) throws IOException { - AtomicReference result = new AtomicReference<>(); - AtomicReference exceptionReference = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "CoordinationMetadata uploaded successfully.")); - result.set(uploadedMetadataAttribute); - }, ex -> { - logger.error( - () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), - ex - ); - exceptionReference.set(ex); - }), - latch); - - writeMetadataComponentAsync(clusterState, COORDINATION_METADATA, COORDINATION_METADATA_FORMAT, clusterState.metadata().coordinationMetadata(), completionListener); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of coordination metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of coordination metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); - } - return result.get(); - } - - private UploadedMetadataAttribute writeSettingsMetadata(ClusterState clusterState) throws IOException { - AtomicReference result = new AtomicReference<>(); - AtomicReference exceptionReference = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "Settings Metadata uploaded successfully.")); - result.set(uploadedMetadataAttribute); - }, ex -> { - logger.error( - () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), - ex - ); - exceptionReference.set(ex); - }), - latch); - - writeMetadataComponentAsync(clusterState, SETTING_METADATA, SETTINGS_METADATA_FORMAT, clusterState.metadata().persistentSettings(), completionListener); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of settings metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of settings metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); - } - return result.get(); - } - - private UploadedMetadataAttribute writeTemplateMetadata(ClusterState clusterState) throws IOException { - AtomicReference result = new AtomicReference<>(); - AtomicReference exceptionReference = new AtomicReference<>(); - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "Templates Metadata uploaded successfully.")); - result.set(uploadedMetadataAttribute); - }, ex -> { - logger.error( - () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), - ex - ); - exceptionReference.set(ex); - }), - latch); - - writeMetadataComponentAsync(clusterState, TEMPLATE_METADATA, TEMPLATE_METADATA_FORMAT, clusterState.metadata().templatesMetadata(), completionListener); - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of templates metadata to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of templates metadata to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); - } - return result.get(); - } - - private Map writeCustomMetadataInParallel(ClusterState clusterState) throws IOException { - Map result = new HashMap<>(); - List exceptionReference = new ArrayList<>(); - - Map customToUpload = clusterState.metadata().customs(); - - final CountDownLatch latch = new CountDownLatch(customToUpload.size()); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap( - (UploadedMetadataAttribute uploadedMetadataAttribute) -> { - logger.trace(String.format(Locale.ROOT, "Custom Metadata uploaded successfully.")); - result.put(uploadedMetadataAttribute.getAttributeName(), uploadedMetadataAttribute); + private UploadedMetadataResults writeMetadataInParallel( + ClusterState clusterState, + List indexToUpload, + Map customToUpload, + boolean uploadCoordinationMetadata, + boolean uploadSettingsMetadata, + boolean uploadTemplateMetadata + ) throws IOException { + int totalUploadTasks = indexToUpload.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata + ? 1 + : 0) + (uploadTemplateMetadata ? 1 : 0); + CountDownLatch latch = new CountDownLatch(totalUploadTasks); + List> uploadTasks = new ArrayList<>(totalUploadTasks); + Map results = new HashMap<>(totalUploadTasks); + List exceptionList = Collections.synchronizedList(new ArrayList<>(totalUploadTasks)); + + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap((ClusterMetadataManifest.UploadedMetadata uploadedMetadata) -> { + logger.trace(String.format(Locale.ROOT, "Metadata component %s uploaded successfully.", uploadedMetadata.getComponent())); + results.put(uploadedMetadata.getComponent(), uploadedMetadata); }, ex -> { logger.error( () -> new ParameterizedMessage("Exception during transfer of Metadata Fragment to Remote {}", ex.getMessage()), ex ); - exceptionReference.add(ex); + exceptionList.add(ex); }), latch ); - for (Map.Entry custom : customToUpload.entrySet()) { - writeMetadataComponentAsync(clusterState, custom.getKey(), CUSTOM_METADATA_FORMAT, custom.getValue(), completionListener); + if (uploadSettingsMetadata) { + uploadTasks.add( + getAsyncMetadataWriteAction( + clusterState, + SETTING_METADATA, + SETTINGS_METADATA_FORMAT, + clusterState.metadata().persistentSettings(), + listener + ) + ); } - - try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of custom metadata to complete - %s", - String.join(", ", customToUpload.keySet()) - ) - ); - exceptionReference.forEach(ex::addSuppressed); - throw ex; - } - } catch (InterruptedException ex) { - exceptionReference.forEach(ex::addSuppressed); - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of custom metadata to complete - %s", - String.join(", ", customToUpload.keySet()) - ), - ex + if (uploadCoordinationMetadata) { + uploadTasks.add( + getAsyncMetadataWriteAction( + clusterState, + COORDINATION_METADATA, + COORDINATION_METADATA_FORMAT, + clusterState.metadata().coordinationMetadata(), + listener + ) ); - Thread.currentThread().interrupt(); - throw exception; } - if (!exceptionReference.isEmpty()) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Exception during transfer of custom metadata to complete - %s", - String.join(", ", customToUpload.keySet()) + if (uploadTemplateMetadata) { + uploadTasks.add( + getAsyncMetadataWriteAction( + clusterState, + TEMPLATES_METADATA, + TEMPLATES_METADATA_FORMAT, + clusterState.metadata().templatesMetadata(), + listener ) ); - exceptionReference.forEach(exception::addSuppressed); - throw exception; } - return result; - } - - /** - * Uploads provided IndexMetadata's to remote store in parallel. The call is blocking so the method waits for upload to finish and then return. - * - * @param clusterState current ClusterState - * @param toUpload list of IndexMetadata to upload - * @return {@code List} list of IndexMetadata uploaded to remote - */ - private List writeIndexMetadataParallel(ClusterState clusterState, List toUpload) - throws IOException { - List exceptionList = Collections.synchronizedList(new ArrayList<>(toUpload.size())); - final CountDownLatch latch = new CountDownLatch(toUpload.size()); - List result = new ArrayList<>(toUpload.size()); - - LatchedActionListener latchedActionListener = new LatchedActionListener<>( - ActionListener.wrap((UploadedIndexMetadata uploadedIndexMetadata) -> { - logger.trace( - String.format(Locale.ROOT, "IndexMetadata uploaded successfully for %s", uploadedIndexMetadata.getIndexName()) - ); - result.add(uploadedIndexMetadata); - }, ex -> { - assert ex instanceof RemoteStateTransferException; - logger.error( - () -> new ParameterizedMessage("Exception during transfer of IndexMetadata to Remote {}", ex.getMessage()), - ex - ); - exceptionList.add(ex); - }), - latch + customToUpload.forEach( + (key, value) -> uploadTasks.add( + getAsyncMetadataWriteAction( + clusterState, + String.join(DELIMITER, CUSTOM_METADATA, key), + CUSTOM_METADATA_FORMAT, + value, + listener + ) + ) ); + indexToUpload.forEach(indexMetadata -> { uploadTasks.add(getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); }); - for (IndexMetadata indexMetadata : toUpload) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX/metadata_4_1690947200 - writeIndexMetadataAsync(clusterState, indexMetadata, latchedActionListener); + // start async upload of all required metadata files + for (CheckedRunnable uploadTask : uploadTasks) { + uploadTask.run(); } try { - if (latch.await(getIndexMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + // TODO: We should add metrics where transfer is timing out. [Issue: #10687] RemoteStateTransferException ex = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of index metadata to complete - %s", - toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining("")) - ) + String.format(Locale.ROOT, "Timed out waiting for transfer of metadata to complete") ); - exceptionList.forEach(ex::addSuppressed); throw ex; } } catch (InterruptedException ex) { - exceptionList.forEach(ex::addSuppressed); RemoteStateTransferException exception = new RemoteStateTransferException( - String.format( - Locale.ROOT, - "Timed out waiting for transfer of index metadata to complete - %s", - toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining("")) - ), + String.format(Locale.ROOT, "Timed out waiting for transfer of metadata to complete - %s"), ex ); Thread.currentThread().interrupt(); throw exception; } - if (exceptionList.size() > 0) { + if (!exceptionList.isEmpty()) { RemoteStateTransferException exception = new RemoteStateTransferException( String.format( Locale.ROOT, - "Exception during transfer of IndexMetadata to Remote %s", - toUpload.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining("")) + "Exception during transfer of following metadata to Remote - %s, %s, %s", + indexToUpload.stream().map(IndexMetadata::getIndexName).collect(Collectors.joining(",")), + customToUpload.keySet().stream().collect(Collectors.joining(", ")), + String.join( + ", ", + (uploadSettingsMetadata ? "settings" : ""), + (uploadCoordinationMetadata ? "coordination" : ""), + (uploadTemplateMetadata ? "templates" : "") + ) ) ); exceptionList.forEach(exception::addSuppressed); throw exception; } - return result; + UploadedMetadataResults response = new UploadedMetadataResults(); + for (Map.Entry entry : results.entrySet()) { + final String name = entry.getKey(); + final ClusterMetadataManifest.UploadedMetadata uploadedMetadata = entry.getValue(); + if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class)) { + response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); + } else if (uploadedMetadata.getComponent().contains(CUSTOM_METADATA)) { + // component name for custom metadata will look like custom__ + response.uploadedCustomMetadataMap.put(name.split(DELIMITER)[1], (UploadedMetadataAttribute) uploadedMetadata); + } else if (COORDINATION_METADATA.equals(uploadedMetadata.getComponent())) { + response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else if (SETTING_METADATA.equals(uploadedMetadata.getComponent())) { + response.uploadedSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else if (TEMPLATES_METADATA.equals(uploadedMetadata.getComponent())) { + response.uploadedTemplatesMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else { + throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); + } + } + return response; } /** @@ -758,11 +603,11 @@ private List writeIndexMetadataParallel(ClusterState clus * @param indexMetadata {@link IndexMetadata} to upload * @param latchedActionListener listener to respond back on after upload finishes */ - private void writeIndexMetadataAsync( + private CheckedRunnable getIndexMetadataAsyncAction( ClusterState clusterState, IndexMetadata indexMetadata, - LatchedActionListener latchedActionListener - ) throws IOException { + LatchedActionListener latchedActionListener + ) { final BlobContainer indexMetadataContainer = indexMetadataContainer( clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), @@ -780,7 +625,7 @@ private void writeIndexMetadataAsync( ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) ); - INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( + return () -> INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( indexMetadata, indexMetadataContainer, indexMetadataFilename, @@ -794,29 +639,23 @@ private void writeIndexMetadataAsync( * Allows async upload of Metadata components to remote */ - private void writeMetadataComponentAsync( + private CheckedRunnable getAsyncMetadataWriteAction( ClusterState clusterState, String component, ChecksumBlobStoreFormat componentMetadataBlobStore, ToXContent componentMetadata, - LatchedActionListener latchedActionListener - ) throws IOException { + LatchedActionListener latchedActionListener + ) { final BlobContainer globalMetadataContainer = globalMetadataContainer( clusterState.getClusterName().value(), clusterState.metadata().clusterUUID() ); final String componentMetadataFilename = metadataAttributeFileName(component, clusterState.metadata().version()); ActionListener completionListener = ActionListener.wrap( - resp -> latchedActionListener.onResponse( - new UploadedMetadataAttribute( - component, - componentMetadataFilename - ) - ), + resp -> latchedActionListener.onResponse(new UploadedMetadataAttribute(component, componentMetadataFilename)), ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, ex)) ); - - componentMetadataBlobStore.writeAsyncWithUrgentPriority( + return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority( componentMetadata, globalMetadataContainer, componentMetadataFilename, @@ -867,18 +706,56 @@ public void start() { blobStoreRepository = (BlobStoreRepository) repository; } + private ClusterMetadataManifest uploadV1Manifest( + ClusterState clusterState, + List uploadedIndexMetadata, + String previousClusterUUID, + String globalMetadataFileName, + boolean committed + ) throws IOException { + synchronized (this) { + final String manifestFileName = getManifestFileName( + clusterState.term(), + clusterState.version(), + committed, + ClusterMetadataManifest.CODEC_V1 + ); + final ClusterMetadataManifest manifest = new ClusterMetadataManifest( + clusterState.term(), + clusterState.getVersion(), + clusterState.metadata().clusterUUID(), + clusterState.stateUUID(), + Version.CURRENT, + nodeId, + committed, + ClusterMetadataManifest.CODEC_V1, + globalMetadataFileName, + uploadedIndexMetadata, + previousClusterUUID, + clusterState.metadata().clusterUUIDCommitted() + ); + writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); + return manifest; + } + } + private ClusterMetadataManifest uploadManifest( ClusterState clusterState, List uploadedIndexMetadata, String previousClusterUUID, UploadedMetadataAttribute uploadedCoordinationMetadata, UploadedMetadataAttribute uploadedSettingsMetadata, - UploadedMetadataAttribute uploadedTemplateMetadata, + UploadedMetadataAttribute uploadedTemplatesMetadata, Map uploadedCustomMetadataMap, boolean committed ) throws IOException { synchronized (this) { - final String manifestFileName = getManifestFileName(clusterState.term(), clusterState.version(), committed); + final String manifestFileName = getManifestFileName( + clusterState.term(), + clusterState.version(), + committed, + MANIFEST_CURRENT_CODEC_VERSION + ); final ClusterMetadataManifest manifest = new ClusterMetadataManifest( clusterState.term(), clusterState.getVersion(), @@ -894,7 +771,7 @@ private ClusterMetadataManifest uploadManifest( clusterState.metadata().clusterUUIDCommitted(), uploadedCoordinationMetadata, uploadedSettingsMetadata, - uploadedTemplateMetadata, + uploadedTemplatesMetadata, uploadedCustomMetadataMap ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); @@ -916,7 +793,7 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); }, ex -> { exceptionReference.set(ex); }), latch); - CLUSTER_METADATA_MANIFEST_FORMAT.writeAsyncWithUrgentPriority( + getClusterMetadataManifestBlobStoreFormat(fileName).writeAsyncWithUrgentPriority( uploadManifest, metadataManifestContainer, fileName, @@ -1006,6 +883,29 @@ private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploa this.metadataManifestUploadTimeout = newMetadataManifestUploadTimeout; } + private Map getUpdatedCustoms(ClusterState currentState, ClusterState previousState) { + if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) { + return new HashMap<>(); + } + Map updatedCustom = new HashMap<>(); + Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); + for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { + if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { + if (!cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { + updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); + } + currentCustoms.remove(cursor.getKey()); + } + } + for (String custom : currentCustoms) { + Metadata.Custom cursor = currentState.metadata().custom(custom); + if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) { + updatedCustom.put(custom, cursor); + } + } + return updatedCustom; + } + public TimeValue getIndexMetadataUploadTimeout() { return this.indexMetadataUploadTimeout; } @@ -1018,7 +918,7 @@ public TimeValue getMetadataManifestUploadTimeout() { return this.metadataManifestUploadTimeout; } - static String getManifestFileName(long term, long version, boolean committed) { + static String getManifestFileName(long term, long version, boolean committed, int codecVersion) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ return String.join( DELIMITER, @@ -1027,8 +927,8 @@ static String getManifestFileName(long term, long version, boolean committed) { RemoteStoreUtils.invertLong(version), (committed ? "C" : "P"), // C for committed and P for published RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(MANIFEST_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last place to - // determine codec version. + String.valueOf(codecVersion) // Keep the codec version at last place only, during read we reads last place to + // determine codec version. ); } @@ -1162,20 +1062,29 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust blobStoreRepository.getNamedXContentRegistry() ); } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { - CoordinationMetadata coordinationMetadata = getCoordinationMetadata(clusterName, clusterUUID, - clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); - Settings settingsMetadata = getSettingsMetadata(clusterName, clusterUUID, - clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); - TemplatesMetadata templatesMetadata = getTemplatesMetadata(clusterName, clusterUUID, - clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); + CoordinationMetadata coordinationMetadata = getCoordinationMetadata( + clusterName, + clusterUUID, + clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() + ); + Settings settingsMetadata = getSettingsMetadata( + clusterName, + clusterUUID, + clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() + ); + TemplatesMetadata templatesMetadata = getTemplatesMetadata( + clusterName, + clusterUUID, + clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() + ); Metadata.Builder builder = new Metadata.Builder(); builder.coordinationMetadata(coordinationMetadata); builder.persistentSettings(settingsMetadata); builder.templates(templatesMetadata); - clusterMetadataManifest.getCustomMetadataMap().forEach( - (key, value) -> - builder.putCustom(key, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key)) - ); + clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> { + String custom = key.split(DELIMITER)[1]; + builder.putCustom(custom, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), custom)); + }); return builder.build(); } else { return Metadata.EMPTY_METADATA; @@ -1235,7 +1144,7 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste // Fetch Templates metadata if (templatesMetadataFileName != null) { String[] splitPath = templatesMetadataFileName.split("/"); - return TEMPLATE_METADATA_FORMAT.read( + return TEMPLATES_METADATA_FORMAT.read( globalMetadataContainer(clusterName, clusterUUID), splitPath[splitPath.length - 1], blobStoreRepository.getNamedXContentRegistry() @@ -1251,11 +1160,16 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste } } - private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, @NonNull String customMetadataFileName, String custom) { + private Metadata.Custom getCustomsMetadata( + String clusterName, + String clusterUUID, + @NonNull String customMetadataFileName, + String custom + ) { try { // Fetch Custom metadata String[] splitPath = customMetadataFileName.split("/"); - ChecksumBlobStoreFormat customChecksumBlobStoreFormat = new ChecksumBlobStoreFormat<>( + ChecksumBlobStoreFormat customChecksumBlobStoreFormat = new ChecksumBlobStoreFormat<>( "custom", METADATA_NAME_FORMAT, (parser -> Metadata.Custom.fromXContent(parser, custom)) @@ -1515,6 +1429,8 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(fileName); if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V1; } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { return CLUSTER_METADATA_MANIFEST_FORMAT_V0; } @@ -1736,4 +1652,34 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa public RemotePersistenceStats getStats() { return remoteStateStats; } + + private static class UploadedMetadataResults { + List uploadedIndexMetadata; + Map uploadedCustomMetadataMap; + UploadedMetadataAttribute uploadedCoordinationMetadata; + UploadedMetadataAttribute uploadedSettingsMetadata; + UploadedMetadataAttribute uploadedTemplatesMetadata; + + public UploadedMetadataResults( + List uploadedIndexMetadata, + Map uploadedCustomMetadataMap, + UploadedMetadataAttribute uploadedCoordinationMetadata, + UploadedMetadataAttribute uploadedSettingsMetadata, + UploadedMetadataAttribute uploadedTemplatesMetadata + ) { + this.uploadedIndexMetadata = uploadedIndexMetadata; + this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; + this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; + this.uploadedSettingsMetadata = uploadedSettingsMetadata; + this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + } + + public UploadedMetadataResults() { + this.uploadedIndexMetadata = new ArrayList<>(); + this.uploadedCustomMetadataMap = new HashMap<>(); + this.uploadedCoordinationMetadata = null; + this.uploadedSettingsMetadata = null; + this.uploadedTemplatesMetadata = null; + } + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6c9a3201656d7..5235dd456af7b 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -16,12 +16,14 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.test.EqualsHashCodeTestUtils; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; public class ClusterMetadataManifestTests extends OpenSearchTestCase { @@ -53,7 +55,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { } } - public void testClusterMetadataManifestXContent() throws IOException { + public void testClusterMetadataManifestXContentV1() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( 1L, @@ -74,6 +76,37 @@ public void testClusterMetadataManifestXContent() throws IOException { originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV1(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + + public void testClusterMetadataManifestXContent() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( + 1L, + 1L, + "test-cluster-uuid", + "test-state-uuid", + Version.CURRENT, + "test-node-id", + false, + ClusterMetadataManifest.CODEC_V2, + null, + Collections.singletonList(uploadedIndexMetadata), + "prev-cluster-uuid", + true, + new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), + new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), + new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), + new HashMap<>() + ); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); assertEquals(originalManifest, fromXContentManifest); @@ -89,11 +122,15 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { Version.CURRENT, "B10RX1f5RJenMQvYccCgSQ", true, - 1, - "test-global-metadata-file", + 2, + null, randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", - true + true, + new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), + new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), + new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), + new HashMap<>() ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 65477051cdb30..510f33210e5c9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -15,7 +15,9 @@ import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; @@ -38,6 +40,7 @@ import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; @@ -46,6 +49,7 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.TestCustomMetadata; import org.opensearch.test.VersionUtils; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; @@ -59,6 +63,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -69,14 +74,19 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; @@ -84,11 +94,14 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; +import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; @@ -223,6 +236,11 @@ public void testWriteFullMetadataSuccess() throws IOException { assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); + assertThat(manifest.getGlobalMetadataFileName(), nullValue()); + assertThat(manifest.getCoordinationMetadata(), notNullValue()); + assertThat(manifest.getSettingsMetadata(), notNullValue()); + assertThat(manifest.getTemplatesMetadata(), notNullValue()); + assertFalse(manifest.getCustomMetadataMap().isEmpty()); } public void testWriteFullMetadataInParallelSuccess() throws IOException { @@ -235,6 +253,18 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { doAnswer((i) -> { actionListenerArgumentCaptor.getValue().onResponse(null); return null; + }).doAnswer((i) -> { + actionListenerArgumentCaptor.getValue().onResponse(null); + return null; + }).doAnswer((i) -> { + actionListenerArgumentCaptor.getValue().onResponse(null); + return null; + }).doAnswer((i) -> { + actionListenerArgumentCaptor.getValue().onResponse(null); + return null; + }).doAnswer((i) -> { + actionListenerArgumentCaptor.getValue().onResponse(null); + return null; }).doAnswer((i) -> { actionListenerArgumentCaptor.getValue().onResponse(null); capturedWriteContext.set(writeContextArgumentCaptor.getValue()); @@ -263,15 +293,19 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName())); assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID())); assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue()); - assertThat(manifest.getGlobalMetadataFileName(), notNullValue()); + assertThat(manifest.getGlobalMetadataFileName(), nullValue()); + assertThat(manifest.getCoordinationMetadata(), notNullValue()); + assertThat(manifest.getSettingsMetadata(), notNullValue()); + assertThat(manifest.getTemplatesMetadata(), notNullValue()); + assertThat(manifest.getCustomMetadataMap().size(), not(0)); assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); assertThat(manifest.getPreviousClusterUUID(), is(expectedManifest.getPreviousClusterUUID())); - assertEquals(actionListenerArgumentCaptor.getAllValues().size(), 3); - assertEquals(writeContextArgumentCaptor.getAllValues().size(), 3); + assertEquals(7, actionListenerArgumentCaptor.getAllValues().size()); + assertEquals(7, writeContextArgumentCaptor.getAllValues().size()); byte[] writtenBytes = capturedWriteContext.get() .getStreamProvider(Integer.MAX_VALUE) @@ -351,7 +385,7 @@ public void testTimeoutWhileWritingManifestFile() throws IOException { remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)); } catch (Exception e) { assertTrue(e instanceof RemoteClusterStateService.RemoteStateTransferException); - assertTrue(e.getMessage().contains("Timed out waiting for transfer of manifest file to complete")); + assertTrue(e.getMessage().contains("Timed out waiting for transfer of metadata to complete")); } } @@ -436,13 +470,28 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { } /* - * Here we will verify the migration of manifest file from codec V0 and V1. + * Here we will verify the migration of manifest file from codec V0. * * Initially codec version is 0 and global metadata is also null, we will perform index metadata update. - * In final manifest codec version should be 1 and - * global metadata should be updated, even if it was not changed in this cluster state update + * In final manifest codec version should be 2 and have metadata files updated, + * even if it was not changed in this cluster state update + */ + public void testMigrationFromCodecV0ManifestToCodecV2Manifest() throws IOException { + verifyCodecMigrationManifest(ClusterMetadataManifest.CODEC_V0); + } + + /* + * Here we will verify the migration of manifest file from codec V1. + * + * Initially codec version is 1 and a global metadata file is there, we will perform index metadata update. + * In final manifest codec version should be 2 and have metadata files updated, + * even if it was not changed in this cluster state update */ - public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOException { + public void testMigrationFromCodecV1ManifestToCodecV2Manifest() throws IOException { + verifyCodecMigrationManifest(ClusterMetadataManifest.CODEC_V1); + } + + private void verifyCodecMigrationManifest(int previousCodec) throws IOException { mockBlobStoreObjects(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) @@ -462,7 +511,7 @@ public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOExcepti // previous manifest with codec 0 and null global metadata final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() - .codecVersion(ClusterMetadataManifest.CODEC_V0) + .codecVersion(previousCodec) .globalMetadataFileName(null) .indices(Collections.emptyList()) .build(); @@ -475,12 +524,28 @@ public void testMigrationFromCodecV0ManifestToCodecV1Manifest() throws IOExcepti ); // global metadata is updated - assertThat(manifestAfterUpdate.getGlobalMetadataFileName(), notNullValue()); + assertThat(manifestAfterUpdate.hasMetadataAttributesFiles(), is(true)); // Manifest file with codec version with 1 is updated. - assertThat(manifestAfterUpdate.getCodecVersion(), is(ClusterMetadataManifest.CODEC_V1)); + assertThat(manifestAfterUpdate.getCodecVersion(), is(MANIFEST_CURRENT_CODEC_VERSION)); } - public void testWriteIncrementalGlobalMetadataSuccess() throws IOException { + public void testWriteIncrementalGlobalMetadataFromCodecV0Success() throws IOException { + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(Collections.emptyList()).build(); + + verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(previousManifest); + } + + public void testWriteIncrementalGlobalMetadataFromCodecV1Success() throws IOException { + final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() + .codecVersion(1) + .globalMetadataFileName("global-metadata-file") + .indices(Collections.emptyList()) + .build(); + + verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(previousManifest); + } + + private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMetadataManifest previousManifest) throws IOException { final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); @@ -488,12 +553,6 @@ public void testWriteIncrementalGlobalMetadataSuccess() throws IOException { .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) .build(); - final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() - .codecVersion(2) - .globalMetadataFileName("global-metadata-file") - .indices(Collections.emptyList()) - .build(); - remoteClusterStateService.start(); final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata( previousClusterState, @@ -502,8 +561,8 @@ public void testWriteIncrementalGlobalMetadataSuccess() throws IOException { ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() + .codecVersion(2) .indices(Collections.emptyList()) - .globalMetadataFileName("mock-filename") .clusterTerm(1L) .stateVersion(1L) .stateUUID("state-uuid") @@ -511,130 +570,239 @@ public void testWriteIncrementalGlobalMetadataSuccess() throws IOException { .previousClusterUUID("prev-cluster-uuid") .build(); - assertThat(manifest.getGlobalMetadataFileName(), notNullValue()); - assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); - assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); - assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); - assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); + assertNull(manifest.getGlobalMetadataFileName()); + assertNotNull(manifest.getCoordinationMetadata()); + assertNotNull(manifest.getSettingsMetadata()); + assertNotNull(manifest.getTemplatesMetadata()); + assertNotEquals(0, manifest.getCustomMetadataMap().size()); + + assertEquals(expectedManifest.getClusterTerm(), manifest.getClusterTerm()); + assertEquals(expectedManifest.getStateVersion(), manifest.getStateVersion()); + assertEquals(expectedManifest.getClusterUUID(), manifest.getClusterUUID()); + assertEquals(expectedManifest.getStateUUID(), manifest.getStateUUID()); + assertEquals(expectedManifest.getCodecVersion(), manifest.getCodecVersion()); } - /* - * Here we will verify index metadata is not uploaded again if change is only in global metadata - */ - public void testGlobalMetadataOnlyUpdated() throws IOException { - // setup - mockBlobStoreObjects(); - final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); - final ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT) - .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata).version(randomNonNegativeLong())) + public void testCoordinationMetadataOnlyUpdated() throws IOException { + // Updating the voting config, as updating the term will upload the full cluster state and other files will also get updated + Function updater = (initialClusterState) -> ClusterState.builder(initialClusterState) + .metadata( + Metadata.builder(initialClusterState.metadata()) + .coordinationMetadata( + CoordinationMetadata.builder(initialClusterState.coordinationMetadata()) + .addVotingConfigExclusion(new CoordinationMetadata.VotingConfigExclusion("excludedNodeId", "excludedNodeName")) + .build() + ) + .build() + ) .build(); - final ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder() - .codecVersion(2) - .globalMetadataFileName("global-metadata-file") - .indices(Collections.emptyList()) + verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> { + // Verify that index metadata information is same in manifest files + assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size()); + IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> { + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName()); + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID()); + // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata + // update + assertEquals( + metadataAfterUpdate.getIndices().get(i).getUploadedFilename(), + initialMetadata.getIndices().get(i).getUploadedFilename() + ); + }); + + // coordination metadata file would have changed + assertFalse( + metadataAfterUpdate.getCoordinationMetadata() + .getUploadedFilename() + .equalsIgnoreCase(initialMetadata.getCoordinationMetadata().getUploadedFilename()) + ); + // Other files will be equal + assertEquals( + metadataAfterUpdate.getSettingsMetadata().getUploadedFilename(), + initialMetadata.getSettingsMetadata().getUploadedFilename() + ); + assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata()); + assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap()); + }); + } + + public void testSettingsMetadataOnlyUpdated() throws IOException { + Function updater = (initialClusterState) -> ClusterState.builder(initialClusterState) + .metadata( + Metadata.builder(initialClusterState.metadata()).persistentSettings(Settings.builder().put("foo", "bar").build()).build() + ) .build(); - remoteClusterStateService.start(); - // Initial cluster state with index. - final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); - // Updating remote cluster state with changing index metadata - final ClusterMetadataManifest manifestAfterIndexMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata( - initialClusterState, - clusterState, - initialManifest - ); + verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> { + // Verify that index metadata information is same in manifest files + assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size()); + IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> { + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName()); + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID()); + // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata + // update + assertEquals( + metadataAfterUpdate.getIndices().get(i).getUploadedFilename(), + initialMetadata.getIndices().get(i).getUploadedFilename() + ); + }); + + // setting metadata file would have changed + assertFalse( + metadataAfterUpdate.getSettingsMetadata() + .getUploadedFilename() + .equalsIgnoreCase(initialMetadata.getSettingsMetadata().getUploadedFilename()) + ); + assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata()); + assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata()); + assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap()); + }); + } - // new cluster state where only global metadata is different - Metadata newMetadata = Metadata.builder(clusterState.metadata()) - .persistentSettings(Settings.builder().put("cluster.blocks.read_only", true).build()) - .version(randomNonNegativeLong()) + public void testTemplatesMetadataOnlyUpdated() throws IOException { + Function updater = (initialClusterState) -> ClusterState.builder(initialClusterState) + .metadata( + Metadata.builder(initialClusterState.metadata()) + .templates( + TemplatesMetadata.builder() + .put( + IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .patterns(Arrays.asList("bar-*", "foo-*")) + .settings( + Settings.builder() + .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)) + .build() + ) + .build() + ) + .build() + ) + .build() + ) .build(); - ClusterState newClusterState = ClusterState.builder(clusterState).metadata(newMetadata).build(); - // updating remote cluster state with global metadata - final ClusterMetadataManifest manifestAfterGlobalMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata( - clusterState, - newClusterState, - manifestAfterIndexMetadataUpdate - ); + verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> { + // Verify that index metadata information is same in manifest files + assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size()); + IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> { + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName()); + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID()); + // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata + // update + assertEquals( + metadataAfterUpdate.getIndices().get(i).getUploadedFilename(), + initialMetadata.getIndices().get(i).getUploadedFilename() + ); + }); - // Verify that index metadata information is same in manifest files - assertThat(manifestAfterIndexMetadataUpdate.getIndices().size(), is(manifestAfterGlobalMetadataUpdate.getIndices().size())); - assertThat( - manifestAfterIndexMetadataUpdate.getIndices().get(0).getIndexName(), - is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getIndexName()) - ); - assertThat( - manifestAfterIndexMetadataUpdate.getIndices().get(0).getIndexUUID(), - is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getIndexUUID()) - ); + // template metadata file would have changed + assertFalse( + metadataAfterUpdate.getTemplatesMetadata() + .getUploadedFilename() + .equalsIgnoreCase(initialMetadata.getTemplatesMetadata().getUploadedFilename()) + ); + assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata()); + assertEquals(metadataAfterUpdate.getSettingsMetadata(), initialMetadata.getSettingsMetadata()); + assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap()); + }); + } - // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata update - assertThat( - manifestAfterIndexMetadataUpdate.getIndices().get(0).getUploadedFilename(), - is(manifestAfterGlobalMetadataUpdate.getIndices().get(0).getUploadedFilename()) - ); + public void testCustomMetadataOnlyUpdated() throws IOException { + Function updater = (initialClusterState) -> ClusterState.builder(initialClusterState) + .metadata( + Metadata.builder(initialClusterState.metadata()) + .putCustom("custom_metadata_type", new CustomMetadata1("mock_custom_metadata")) + .build() + ) + .build(); - // global metadata file would have changed - assertFalse( - manifestAfterIndexMetadataUpdate.getGlobalMetadataFileName() - .equalsIgnoreCase(manifestAfterGlobalMetadataUpdate.getGlobalMetadataFileName()) - ); + verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> { + // Verify that index metadata information is same in manifest files + assertEquals(metadataAfterUpdate.getIndices().size(), initialMetadata.getIndices().size()); + IntStream.range(0, initialMetadata.getIndices().size()).forEach(i -> { + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexName(), initialMetadata.getIndices().get(i).getIndexName()); + assertEquals(metadataAfterUpdate.getIndices().get(i).getIndexUUID(), initialMetadata.getIndices().get(i).getIndexUUID()); + // since timestamp is part of file name, if file name is same we can confirm that file is not update in global metadata + // update + assertEquals( + metadataAfterUpdate.getIndices().get(i).getUploadedFilename(), + initialMetadata.getIndices().get(i).getUploadedFilename() + ); + // custom metadata map would have changed + assertNotEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap()); + assertEquals(initialMetadata.getCustomMetadataMap().size() + 1, metadataAfterUpdate.getCustomMetadataMap().size()); + initialMetadata.getCustomMetadataMap().forEach((k, v) -> { + assertTrue(metadataAfterUpdate.getCustomMetadataMap().containsKey(k)); + assertEquals(v, metadataAfterUpdate.getCustomMetadataMap().get(k)); + }); + assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata()); + assertEquals(metadataAfterUpdate.getSettingsMetadata(), initialMetadata.getSettingsMetadata()); + assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata()); + }); + }); } /* * Here we will verify global metadata is not uploaded again if change is only in index metadata */ public void testIndexMetadataOnlyUpdated() throws IOException { - // setup - mockBlobStoreObjects(); - final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); - final ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT) - .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) - .build(); - final ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder() - .codecVersion(2) - .indices(Collections.emptyList()) + Function updater = (initialState) -> ClusterState.builder(initialState) + .metadata( + Metadata.builder(initialState.metadata()) + .put( + IndexMetadata.builder("test" + randomAlphaOfLength(3)) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ) + .numberOfShards(1) + .numberOfReplicas(0) + ) + .build() + ) .build(); - remoteClusterStateService.start(); - // Initial cluster state with global metadata. - final ClusterState clusterState = generateClusterStateWithGlobalMetadata().nodes(nodesWithLocalNodeClusterManager()).build(); + verifyMetadataAttributeOnlyUpdated(updater, (initialMetadata, metadataAfterUpdate) -> { + assertEquals(metadataAfterUpdate.getCoordinationMetadata(), initialMetadata.getCoordinationMetadata()); + assertEquals(metadataAfterUpdate.getSettingsMetadata(), initialMetadata.getSettingsMetadata()); + assertEquals(metadataAfterUpdate.getTemplatesMetadata(), initialMetadata.getTemplatesMetadata()); + assertEquals(metadataAfterUpdate.getCustomMetadataMap(), initialMetadata.getCustomMetadataMap()); + assertEquals(initialMetadata.getIndices().size() + 1, metadataAfterUpdate.getIndices().size()); + }); + } - // Updating remote cluster state with changing global metadata - final ClusterMetadataManifest manifestAfterGlobalMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata( - initialClusterState, - clusterState, - initialManifest - ); + /* + * Here we will verify index metadata is not uploaded again if change is only in global metadata + */ + private void verifyMetadataAttributeOnlyUpdated( + Function clusterStateUpdater, + BiConsumer assertions + ) throws IOException { + // setup + mockBlobStoreObjects(); - // new cluster state where only Index metadata is different - final IndexMetadata indexMetadata = new IndexMetadata.Builder("test").settings( - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") - .build() - ).numberOfShards(1).numberOfReplicas(0).build(); - Metadata newMetadata = Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build(); - ClusterState newClusterState = ClusterState.builder(clusterState).metadata(newMetadata).build(); + // Initial cluster state with index. + final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_"); - // updating remote cluster state with index metadata - final ClusterMetadataManifest manifestAfterIndexMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata( - clusterState, - newClusterState, - manifestAfterGlobalMetadataUpdate - ); + ClusterState newClusterState = clusterStateUpdater.apply(initialClusterState); - // Verify that global metadata information is same in manifest files after updating index Metadata - // since timestamp is part of file name, if file name is same we can confirm that file is not update in index metadata update - assertThat( - manifestAfterIndexMetadataUpdate.getGlobalMetadataFileName(), - is(manifestAfterGlobalMetadataUpdate.getGlobalMetadataFileName()) - ); + // updating remote cluster state with global metadata + final ClusterMetadataManifest manifestAfterMetadataUpdate; + if (initialClusterState.term() == newClusterState.term()) { + manifestAfterMetadataUpdate = remoteClusterStateService.writeIncrementalMetadata( + initialClusterState, + newClusterState, + initialManifest + ); + } else { + manifestAfterMetadataUpdate = remoteClusterStateService.writeFullMetadata(newClusterState, initialClusterState.stateUUID()); + } - // Index metadata would have changed - assertThat(manifestAfterGlobalMetadataUpdate.getIndices().size(), is(0)); - assertThat(manifestAfterIndexMetadataUpdate.getIndices().size(), is(1)); + assertions.accept(initialManifest, manifestAfterMetadataUpdate); } public void testReadLatestMetadataManifestFailedIOException() throws IOException { @@ -795,7 +963,10 @@ public void testReadGlobalMetadata() throws IOException { .stateUUID("state-uuid") .clusterUUID("cluster-uuid") .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) - .globalMetadataFileName("global-metadata-file") + .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) + .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) + .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) + .put(IndexGraveyard.TYPE, new UploadedMetadataAttribute(CUSTOM_METADATA + DELIMITER + IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") @@ -830,7 +1001,7 @@ public void testReadGlobalMetadataIOException() throws IOException { .stateVersion(1L) .stateUUID("state-uuid") .clusterUUID("cluster-uuid") - .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .codecVersion(ClusterMetadataManifest.CODEC_V1) .globalMetadataFileName(globalIndexMetadataName) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) @@ -1124,18 +1295,24 @@ public void testFileNames() { assertThat(splittedIndexMetadataFileName[1], is(RemoteStoreUtils.invertLong(indexMetadata.getVersion()))); assertThat(splittedIndexMetadataFileName[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION))); + verifyManifestFileNameWithCodec(MANIFEST_CURRENT_CODEC_VERSION); + verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V1); + verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V0); + } + + private void verifyManifestFileNameWithCodec(int codecVersion) { int term = randomIntBetween(5, 10); int version = randomIntBetween(5, 10); - String manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, true); + String manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, true, codecVersion); assertThat(manifestFileName.split(DELIMITER).length, is(6)); String[] splittedName = manifestFileName.split(DELIMITER); assertThat(splittedName[0], is(MANIFEST_FILE_PREFIX)); assertThat(splittedName[1], is(RemoteStoreUtils.invertLong(term))); assertThat(splittedName[2], is(RemoteStoreUtils.invertLong(version))); assertThat(splittedName[3], is("C")); - assertThat(splittedName[5], is(String.valueOf(MANIFEST_CURRENT_CODEC_VERSION))); + assertThat(splittedName[5], is(String.valueOf(codecVersion))); - manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, false); + manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, false, codecVersion); splittedName = manifestFileName.split(DELIMITER); assertThat(splittedName[3], is("P")); } @@ -1251,12 +1428,16 @@ private void mockObjectsForGettingPreviousClusterUUID( new UploadedIndexMetadata("index1", "index-uuid1", "key1"), new UploadedIndexMetadata("index2", "index-uuid2", "key2") ); + Map customMetadataMap = new HashMap<>(); final ClusterMetadataManifest clusterManifest1 = generateClusterMetadataManifest( "cluster-uuid1", clusterUUIDsPointers.get("cluster-uuid1"), randomAlphaOfLength(10), uploadedIndexMetadataList1, - "test-metadata1", + customMetadataMap, + new UploadedMetadataAttribute(COORDINATION_METADATA, "key3"), + new UploadedMetadataAttribute(SETTING_METADATA, "key4"), + new UploadedMetadataAttribute(TEMPLATES_METADATA, "key5"), clusterUUIDCommitted.getOrDefault("cluster-uuid1", true) ); Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); @@ -1275,7 +1456,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2); mockBlobContainerForGlobalMetadata(blobContainer1, clusterManifest1, metadata1); - mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V1); + mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V2); List uploadedIndexMetadataList2 = List.of( new UploadedIndexMetadata("index1", "index-uuid1", "key1"), @@ -1286,7 +1467,10 @@ private void mockObjectsForGettingPreviousClusterUUID( clusterUUIDsPointers.get("cluster-uuid2"), randomAlphaOfLength(10), uploadedIndexMetadataList2, - "test-metadata2", + customMetadataMap, + new UploadedMetadataAttribute(COORDINATION_METADATA, "key3"), + new UploadedMetadataAttribute(SETTING_METADATA, "key4"), + new UploadedMetadataAttribute(TEMPLATES_METADATA, "key5"), clusterUUIDCommitted.getOrDefault("cluster-uuid2", true) ); IndexMetadata indexMetadata3 = IndexMetadata.builder("index1") @@ -1304,7 +1488,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4); mockBlobContainerForGlobalMetadata(blobContainer2, clusterManifest2, metadata2); - mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V1); + mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V2); // differGlobalMetadata controls which one of IndexMetadata or Metadata object would be different // when comparing cluster-uuid3 and cluster-uuid1 state. @@ -1326,17 +1510,19 @@ private void mockObjectsForGettingPreviousClusterUUID( Metadata metadata3 = Metadata.builder() .persistentSettings(Settings.builder().put(Metadata.SETTING_READ_ONLY_SETTING.getKey(), !differGlobalMetadata).build()) .build(); - final ClusterMetadataManifest clusterManifest3 = generateClusterMetadataManifest( "cluster-uuid3", clusterUUIDsPointers.get("cluster-uuid3"), randomAlphaOfLength(10), uploadedIndexMetadataList3, - "test-metadata3", + customMetadataMap, + new UploadedMetadataAttribute(COORDINATION_METADATA, "key3"), + new UploadedMetadataAttribute(SETTING_METADATA, "key4"), + new UploadedMetadataAttribute(TEMPLATES_METADATA, "key5"), clusterUUIDCommitted.getOrDefault("cluster-uuid3", true) ); mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3); - mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V1); + mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V2); ArrayList mockBlobContainerOrderedList = new ArrayList<>( List.of(blobContainer1, blobContainer1, blobContainer3, blobContainer3, blobContainer2, blobContainer2) @@ -1356,7 +1542,7 @@ private void mockObjectsForGettingPreviousClusterUUID( when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); } - private ClusterMetadataManifest generateClusterMetadataManifest( + private ClusterMetadataManifest generateV1ClusterMetadataManifest( String clusterUUID, String previousClusterUUID, String stateUUID, @@ -1380,6 +1566,36 @@ private ClusterMetadataManifest generateClusterMetadataManifest( .build(); } + private ClusterMetadataManifest generateClusterMetadataManifest( + String clusterUUID, + String previousClusterUUID, + String stateUUID, + List uploadedIndexMetadata, + Map customMetadataMap, + UploadedMetadataAttribute coordinationMetadata, + UploadedMetadataAttribute settingsMetadata, + UploadedMetadataAttribute templatesMetadata, + Boolean isUUIDCommitted + ) { + return ClusterMetadataManifest.builder() + .indices(uploadedIndexMetadata) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID(stateUUID) + .clusterUUID(clusterUUID) + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID(previousClusterUUID) + .committed(true) + .clusterUUIDCommitted(isUUIDCommitted) + .coordinationMetadata(coordinationMetadata) + .settingMetadata(settingsMetadata) + .templatesMetadata(templatesMetadata) + .customMetadataMap(customMetadataMap) + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .build(); + } + private BlobContainer mockBlobStoreObjects() { return mockBlobStoreObjects(BlobContainer.class); } @@ -1419,7 +1635,7 @@ private void mockBlobContainer( int codecVersion ) throws IOException { String manifestFileName = codecVersion >= ClusterMetadataManifest.CODEC_V1 - ? "manifest__manifestFileName__abcd__abcd__abcd__1" + ? "manifest__manifestFileName__abcd__abcd__abcd__" + codecVersion : "manifestFileName"; BlobMetadata blobMetadata = new PlainBlobMetadata(manifestFileName, 1); when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) @@ -1460,7 +1676,8 @@ private void mockBlobContainerForGlobalMetadata( ClusterMetadataManifest clusterMetadataManifest, Metadata metadata ) throws IOException { - String mockManifestFileName = "manifest__1__2__C__456__1"; + int codecVersion = clusterMetadataManifest.getCodecVersion(); + String mockManifestFileName = "manifest__1__2__C__456__" + codecVersion; BlobMetadata blobMetadata = new PlainBlobMetadata(mockManifestFileName, 1); when( blobContainer.listBlobsByPrefixInSortedOrder( @@ -1477,19 +1694,84 @@ private void mockBlobContainerForGlobalMetadata( FORMAT_PARAMS ); when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes())); + if (codecVersion >= ClusterMetadataManifest.CODEC_V2) { + String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); + when(blobContainer.readBlob(RemoteClusterStateService.COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))).thenAnswer( + (invocationOnMock) -> { + BytesReference bytesReference = RemoteClusterStateService.COORDINATION_METADATA_FORMAT.serialize( + metadata.coordinationMetadata(), + coordinationFileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes()); + } + ); + + String settingsFileName = getFileNameFromPath(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); + when(blobContainer.readBlob(RemoteClusterStateService.SETTINGS_METADATA_FORMAT.blobName(settingsFileName))).thenAnswer( + (invocationOnMock) -> { + BytesReference bytesReference = RemoteClusterStateService.SETTINGS_METADATA_FORMAT.serialize( + metadata.persistentSettings(), + settingsFileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes()); + } + ); - String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); - when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))).thenAnswer( - (invocationOnMock) -> { - BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize( - metadata, - "global-metadata-file", - blobStoreRepository.getCompressor(), - FORMAT_PARAMS + String templatesFileName = getFileNameFromPath(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); + when(blobContainer.readBlob(RemoteClusterStateService.TEMPLATES_METADATA_FORMAT.blobName(templatesFileName))).thenAnswer( + (invocationOnMock) -> { + BytesReference bytesReference = RemoteClusterStateService.TEMPLATES_METADATA_FORMAT.serialize( + metadata.templatesMetadata(), + templatesFileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes()); + } + ); + + Map customFileMap = clusterMetadataManifest.getCustomMetadataMap() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> getFileNameFromPath(entry.getValue().getUploadedFilename()))); + + for (Map.Entry entry : customFileMap.entrySet()) { + String custom = entry.getKey(); + String fileName = entry.getValue(); + when(blobContainer.readBlob(RemoteClusterStateService.CUSTOM_METADATA_FORMAT.blobName(fileName))).thenAnswer( + (invocation) -> { + BytesReference bytesReference = RemoteClusterStateService.CUSTOM_METADATA_FORMAT.serialize( + metadata.custom(custom), + fileName, + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes()); + } ); - return new ByteArrayInputStream(bytesGlobalMetadata.streamInput().readAllBytes()); } - ); + } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); + when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))) + .thenAnswer((invocationOnMock) -> { + BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize( + metadata, + "global-metadata-file", + blobStoreRepository.getCompressor(), + FORMAT_PARAMS + ); + return new ByteArrayInputStream(bytesGlobalMetadata.streamInput().readAllBytes()); + }); + } + } + + private String getFileNameFromPath(String filePath) { + String[] splitPath = filePath.split("/"); + return splitPath[splitPath.length - 1]; } private static ClusterState.Builder generateClusterStateWithGlobalMetadata() { @@ -1519,7 +1801,9 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() { .numberOfReplicas(0) .build(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); - + final Settings settings = Settings.builder().put("mock-settings", true).build(); + final TemplatesMetadata templatesMetadata = TemplatesMetadata.EMPTY_METADATA; + final CustomMetadata1 customMetadata1 = new CustomMetadata1("custom-metadata-1"); return ClusterState.builder(ClusterName.DEFAULT) .version(1L) .stateUUID("state-uuid") @@ -1529,6 +1813,9 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() { .put(indexMetadata, true) .clusterUUID("cluster-uuid") .coordinationMetadata(coordinationMetadata) + .persistentSettings(settings) + .templates(templatesMetadata) + .putCustom(customMetadata1.getWriteableName(), customMetadata1) .build() ); } @@ -1537,4 +1824,27 @@ private static DiscoveryNodes nodesWithLocalNodeClusterManager() { return DiscoveryNodes.builder().clusterManagerNodeId("cluster-manager-id").localNodeId("cluster-manager-id").build(); } + private static class CustomMetadata1 extends TestCustomMetadata { + public static final String TYPE = "custom_md_1"; + + CustomMetadata1(String data) { + super(data); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT; + } + + @Override + public EnumSet context() { + return EnumSet.of(Metadata.XContentContext.GATEWAY); + } + } + } From 279dbbe24de4283e7a93565a7e2f6483f90d6c88 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 20 Feb 2024 18:18:25 +0530 Subject: [PATCH 003/133] Added javadocs Signed-off-by: Shivansh Arora --- .../cluster/metadata/TemplatesMetadata.java | 10 +++++++++ .../remote/ClusterMetadataManifest.java | 22 +++++++++++++++++-- .../remote/RemoteClusterStateService.java | 20 ++++++++--------- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 179d5aa6fbc52..7fc5ba4e9ad5e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -20,6 +20,11 @@ import java.util.Map; import java.util.Objects; +/** + * Metadata for legacy templates + * + * @opensearch.internal + */ public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment { public static TemplatesMetadata EMPTY_METADATA = builder().build(); private final Map templates; @@ -73,6 +78,11 @@ public int hashCode() { return templates != null ? templates.hashCode() : 0; } + /** + * Builder for the templates metadata + * + * @opensearch.api + */ public static class Builder { private final Map templates; diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index a3a84924e4563..34df404b98313 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -227,7 +227,7 @@ private static void declareParser(ConstructingObjectParser entry : results.entrySet()) { - final String name = entry.getKey(); - final ClusterMetadataManifest.UploadedMetadata uploadedMetadata = entry.getValue(); + results.forEach((name, uploadedMetadata) -> { if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class)) { response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); } else if (uploadedMetadata.getComponent().contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom__ - response.uploadedCustomMetadataMap.put(name.split(DELIMITER)[1], (UploadedMetadataAttribute) uploadedMetadata); + String custom = name.split(DELIMITER)[1]; + response.uploadedCustomMetadataMap.put(custom, + new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename())); } else if (COORDINATION_METADATA.equals(uploadedMetadata.getComponent())) { response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (SETTING_METADATA.equals(uploadedMetadata.getComponent())) { @@ -592,7 +592,7 @@ private UploadedMetadataResults writeMetadataInParallel( } else { throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); } - } + }); return response; } @@ -957,7 +957,7 @@ private static String globalMetadataFileName(Metadata metadata) { } private static String metadataAttributeFileName(String componentPrefix, Long metadataVersion) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/settings______ + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ return String.join( DELIMITER, componentPrefix, @@ -1081,10 +1081,8 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust builder.coordinationMetadata(coordinationMetadata); builder.persistentSettings(settingsMetadata); builder.templates(templatesMetadata); - clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> { - String custom = key.split(DELIMITER)[1]; - builder.putCustom(custom, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), custom)); - }); + clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> + builder.putCustom(key, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key))); return builder.build(); } else { return Metadata.EMPTY_METADATA; @@ -1099,7 +1097,7 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust private CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { try { - // Fetch Coordintaion metadata + // Fetch Coordination metadata if (coordinationMetadataFileName != null) { String[] splitPath = coordinationMetadataFileName.split("/"); return COORDINATION_METADATA_FORMAT.read( From 054284959fbe7aa95b1788617bc154be4b771823 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 29 Feb 2024 06:07:44 +0530 Subject: [PATCH 004/133] Address PR comment Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 30 +++++++++++++++++++ .../remote/ClusterMetadataManifest.java | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index dfde1b958882c..d5fe07d0525bc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -8,22 +8,30 @@ package org.opensearch.gateway.remote; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.DiscoveryStats; +import org.opensearch.monitor.fs.FsInfo; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchIntegTestCase; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Base64; import java.util.Map; import java.util.stream.Collectors; +import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; @@ -181,6 +189,28 @@ public void testRemoteStateStatsFromAllNodes() { } } + public void testRemoteRestoreOnClusterManagerRestartCorruptedLocal() throws IOException { + initialTestSetup(1, 0, 1, 1); + String clusterManager = internalCluster().getClusterManagerName(); + NodesStatsResponse response = client().admin() + .cluster().prepareNodesStats(clusterManager).addMetric(FS.metricName()).get(); + +// internalCluster().stopCurrentClusterManagerNode(); +// for (FsInfo.Path info : response.getNodes().get(0).getFs()) { +// String path = info.getPath(); +// Path state = PathUtils.get(path).resolve("_state"); +// if (Files.exists(state)) { +// try (Directory dir = FSDirectory.open(state)) { +// for (String file : dir.listAll()) { +// dir.deleteFile(file); +// } +// } +// } +// } + internalCluster().startClusterManagerOnlyNode(); + ensureStableCluster(2); + } + private void validateNodesStatsResponse(NodesStatsResponse nodesStatsResponse) { // _nodes/stats/discovery must never fail due to any exception assertFalse(nodesStatsResponse.toString().contains("exception")); diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 34df404b98313..a1bba93477210 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -209,7 +209,7 @@ private static void declareParser(ConstructingObjectParser= CODEC_V2) { parser.declareInt(ConstructingObjectParser.constructorArg(), CODEC_VERSION_FIELD); parser.declareNamedObject( ConstructingObjectParser.optionalConstructorArg(), From c9fb25bb123085669e08d9600d673d14d68e2274 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 13 Mar 2024 01:16:25 +0530 Subject: [PATCH 005/133] Add Integ test Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 54 +++++++++----- .../remote/RemoteClusterStateService.java | 73 +++++++++++-------- .../RemoteClusterStateServiceTests.java | 3 +- 3 files changed, 80 insertions(+), 50 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index d5fe07d0525bc..9f7b15ba2ed47 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -29,11 +29,20 @@ import java.nio.file.Path; import java.util.Base64; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase { @@ -191,24 +200,33 @@ public void testRemoteStateStatsFromAllNodes() { public void testRemoteRestoreOnClusterManagerRestartCorruptedLocal() throws IOException { initialTestSetup(1, 0, 1, 1); - String clusterManager = internalCluster().getClusterManagerName(); - NodesStatsResponse response = client().admin() - .cluster().prepareNodesStats(clusterManager).addMetric(FS.metricName()).get(); - -// internalCluster().stopCurrentClusterManagerNode(); -// for (FsInfo.Path info : response.getNodes().get(0).getFs()) { -// String path = info.getPath(); -// Path state = PathUtils.get(path).resolve("_state"); -// if (Files.exists(state)) { -// try (Directory dir = FSDirectory.open(state)) { -// for (String file : dir.listAll()) { -// dir.deleteFile(file); -// } -// } -// } -// } - internalCluster().startClusterManagerOnlyNode(); - ensureStableCluster(2); + + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); + BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); + BlobPath globalMetadataPath = repository.basePath().add( + Base64.getUrlEncoder().withoutPadding() + .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) + ).add("cluster-state").add(getClusterState().metadata().clusterUUID()).add("global-metadata"); + + Map metadataFiles = repository.blobStore().blobContainer(globalMetadataPath).listBlobs() + .keySet().stream() + .map(fileName -> { + logger.info(fileName); + return fileName.split(DELIMITER)[0]; + }) + .collect(Collectors.toMap(Function.identity(), key -> 1, Integer::sum)); + + assertTrue(metadataFiles.containsKey(COORDINATION_METADATA)); + assertEquals(1, (int) metadataFiles.get(COORDINATION_METADATA)); + assertTrue(metadataFiles.containsKey(SETTING_METADATA)); + assertEquals(1, (int) metadataFiles.get(SETTING_METADATA)); + assertTrue(metadataFiles.containsKey(TEMPLATES_METADATA)); + assertEquals(1, (int) metadataFiles.get(TEMPLATES_METADATA)); + assertTrue(metadataFiles.keySet().stream().anyMatch(key -> key.startsWith(CUSTOM_METADATA))); + assertFalse(metadataFiles.containsKey(METADATA_FILE_PREFIX)); } private void validateNodesStatsResponse(NodesStatsResponse nodesStatsResponse) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 41895d4312810..9d6cb4fcf48ef 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -66,8 +66,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import reactor.util.annotation.NonNull; - +import static java.util.Objects.requireNonNull; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; @@ -85,6 +84,7 @@ public class RemoteClusterStateService implements Closeable { public static final int RETAINED_MANIFESTS = 10; public static final String DELIMITER = "__"; + public static final String CUSTOM_DELIMITER = "--"; private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); @@ -471,7 +471,7 @@ private UploadedMetadataResults writeMetadataInParallel( ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalUploadTasks); - List> uploadTasks = new ArrayList<>(totalUploadTasks); + Map> uploadTasks = new HashMap<>(totalUploadTasks); Map results = new HashMap<>(totalUploadTasks); List exceptionList = Collections.synchronizedList(new ArrayList<>(totalUploadTasks)); @@ -490,7 +490,8 @@ private UploadedMetadataResults writeMetadataInParallel( ); if (uploadSettingsMetadata) { - uploadTasks.add( + uploadTasks.put( + SETTING_METADATA, getAsyncMetadataWriteAction( clusterState, SETTING_METADATA, @@ -501,7 +502,8 @@ private UploadedMetadataResults writeMetadataInParallel( ); } if (uploadCoordinationMetadata) { - uploadTasks.add( + uploadTasks.put( + COORDINATION_METADATA, getAsyncMetadataWriteAction( clusterState, COORDINATION_METADATA, @@ -512,7 +514,8 @@ private UploadedMetadataResults writeMetadataInParallel( ); } if (uploadTemplateMetadata) { - uploadTasks.add( + uploadTasks.put( + TEMPLATES_METADATA, getAsyncMetadataWriteAction( clusterState, TEMPLATES_METADATA, @@ -523,20 +526,24 @@ private UploadedMetadataResults writeMetadataInParallel( ); } customToUpload.forEach( - (key, value) -> uploadTasks.add( - getAsyncMetadataWriteAction( - clusterState, - String.join(DELIMITER, CUSTOM_METADATA, key), - CUSTOM_METADATA_FORMAT, - value, - listener - ) - ) + (key, value) -> { + String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key); + uploadTasks.put( + customComponent, + getAsyncMetadataWriteAction( + clusterState, + customComponent, + CUSTOM_METADATA_FORMAT, + value, + listener + ) + ); + } ); - indexToUpload.forEach(indexMetadata -> { uploadTasks.add(getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); }); + indexToUpload.forEach(indexMetadata -> { uploadTasks.put(indexMetadata.getIndexName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); }); // start async upload of all required metadata files - for (CheckedRunnable uploadTask : uploadTasks) { + for (CheckedRunnable uploadTask : uploadTasks.values()) { uploadTask.run(); } @@ -544,13 +551,23 @@ private UploadedMetadataResults writeMetadataInParallel( if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { // TODO: We should add metrics where transfer is timing out. [Issue: #10687] RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of metadata to complete") + String.format( + Locale.ROOT, + "Timed out waiting for transfer of following metadata to complete - %s", + String.join(", ", uploadTasks.keySet()) + ) ); + exceptionList.forEach(ex::addSuppressed); throw ex; } } catch (InterruptedException ex) { + exceptionList.forEach(ex::addSuppressed); RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of metadata to complete - %s"), + String.format( + Locale.ROOT, + "Timed out waiting for transfer of metadata to complete - %s", + String.join(", ", uploadTasks.keySet()) + ), ex ); Thread.currentThread().interrupt(); @@ -560,15 +577,8 @@ private UploadedMetadataResults writeMetadataInParallel( RemoteStateTransferException exception = new RemoteStateTransferException( String.format( Locale.ROOT, - "Exception during transfer of following metadata to Remote - %s, %s, %s", - indexToUpload.stream().map(IndexMetadata::getIndexName).collect(Collectors.joining(",")), - customToUpload.keySet().stream().collect(Collectors.joining(", ")), - String.join( - ", ", - (uploadSettingsMetadata ? "settings" : ""), - (uploadCoordinationMetadata ? "coordination" : ""), - (uploadTemplateMetadata ? "templates" : "") - ) + "Exception during transfer of following metadata to Remote - %s", + String.join(", ", uploadTasks.keySet()) ) ); exceptionList.forEach(exception::addSuppressed); @@ -579,8 +589,8 @@ private UploadedMetadataResults writeMetadataInParallel( if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class)) { response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); } else if (uploadedMetadata.getComponent().contains(CUSTOM_METADATA)) { - // component name for custom metadata will look like custom__ - String custom = name.split(DELIMITER)[1]; + // component name for custom metadata will look like custom-- + String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; response.uploadedCustomMetadataMap.put(custom, new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename())); } else if (COORDINATION_METADATA.equals(uploadedMetadata.getComponent())) { @@ -1161,9 +1171,10 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste private Metadata.Custom getCustomsMetadata( String clusterName, String clusterUUID, - @NonNull String customMetadataFileName, + String customMetadataFileName, String custom ) { + requireNonNull(customMetadataFileName); try { // Fetch Custom metadata String[] splitPath = customMetadataFileName.split("/"); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 510f33210e5c9..c3fd7098174f6 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -86,6 +86,7 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; @@ -966,7 +967,7 @@ public void testReadGlobalMetadata() throws IOException { .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) - .put(IndexGraveyard.TYPE, new UploadedMetadataAttribute(CUSTOM_METADATA + DELIMITER + IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) + .put(IndexGraveyard.TYPE, new UploadedMetadataAttribute(CUSTOM_METADATA + CUSTOM_DELIMITER + IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") From fc270d16a9dc770ebe747a4d8da5ce248c5911a5 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 13 Mar 2024 15:23:26 +0530 Subject: [PATCH 006/133] Added a bwc test Signed-off-by: Shivansh Arora --- .../opensearch/upgrades/ClusterStateIT.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java new file mode 100644 index 0000000000000..2d606d27a34e0 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.upgrades; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; + +import java.util.Map; + +public class ClusterStateIT extends AbstractRollingTestCase{ + public void testTemplateMetadataUpgrades() throws Exception { + if (CLUSTER_TYPE == ClusterType.OLD) { + String templateName = "my_template"; + Request putIndexTemplate = new Request("PUT", "_template/" + templateName); + putIndexTemplate.setJsonEntity("{\"index_patterns\": [\"pattern-1\", \"log-*\"]}"); + client().performRequest(putIndexTemplate); + verifyTemplateMetadataInClusterState(); + } else { + verifyTemplateMetadataInClusterState(); + } + } + + @SuppressWarnings("unchecked") + private static void verifyTemplateMetadataInClusterState() throws Exception { + Request request = new Request("GET", "_cluster/state/metadata"); + Response response = client().performRequest(request); + assertOK(response); + Map metadata = (Map) entityAsMap(response).get("metadata"); + assertNotNull(metadata.get("templates")); + } +} From adb4cf2d8bc1ecf11a3a565595b96868c8abc849 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 14 Mar 2024 16:06:01 +0530 Subject: [PATCH 007/133] Address PR comments Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 33 ++++++------ .../opensearch/cluster/metadata/Metadata.java | 8 +-- .../cluster/metadata/TemplatesMetadata.java | 11 ++-- .../remote/RemoteClusterStateService.java | 50 +++++++++---------- .../remote/ClusterMetadataManifestTests.java | 43 ++++++++++++++-- 5 files changed, 84 insertions(+), 61 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index 9f7b15ba2ed47..61b34af5be3ba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -8,16 +8,12 @@ package org.opensearch.gateway.remote; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.DiscoveryStats; -import org.opensearch.monitor.fs.FsInfo; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -25,16 +21,11 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Base64; import java.util.Map; -import java.util.Optional; -import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA; @@ -198,7 +189,7 @@ public void testRemoteStateStatsFromAllNodes() { } } - public void testRemoteRestoreOnClusterManagerRestartCorruptedLocal() throws IOException { + public void testRemoteClusterStateMetadataSplit() throws IOException { initialTestSetup(1, 0, 1, 1); RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( @@ -206,13 +197,21 @@ public void testRemoteRestoreOnClusterManagerRestartCorruptedLocal() throws IOEx ); RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); - BlobPath globalMetadataPath = repository.basePath().add( - Base64.getUrlEncoder().withoutPadding() - .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) - ).add("cluster-state").add(getClusterState().metadata().clusterUUID()).add("global-metadata"); - - Map metadataFiles = repository.blobStore().blobContainer(globalMetadataPath).listBlobs() - .keySet().stream() + BlobPath globalMetadataPath = repository.basePath() + .add( + Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) + ) + .add("cluster-state") + .add(getClusterState().metadata().clusterUUID()) + .add("global-metadata"); + + Map metadataFiles = repository.blobStore() + .blobContainer(globalMetadataPath) + .listBlobs() + .keySet() + .stream() .map(fileName -> { logger.info(fileName); return fileName.split(DELIMITER)[0]; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 0141a74d01d0c..222ecee5ad376 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -188,13 +188,7 @@ static Custom fromXContent(XContentParser parser, String name) throws IOExceptio static Custom fromXContent(XContentParser parser) throws IOException { String currentFieldName = parser.currentName(); - try { - return parser.namedObject(Custom.class, currentFieldName, null); - } catch (NamedObjectNotFoundException e) { - logger.warn("Unknown custom object with type {}", currentFieldName); - parser.skipChildren(); - throw e; - } + return fromXContent(parser, currentFieldName); } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 7fc5ba4e9ad5e..36205be4d7e08 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -94,8 +94,8 @@ public Builder(Map templates) { this.templates = templates; } - public Builder put(IndexTemplateMetadata.Builder template) { - return put(template.build()); + public Builder put(IndexTemplateMetadata.Builder templateBuilder) { + return put(templateBuilder.build()); } public Builder put(IndexTemplateMetadata template) { @@ -117,18 +117,15 @@ public TemplatesMetadata build() { return new TemplatesMetadata(templates); } - public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException { - // builder.startObject("templates-metadata"); - for (IndexTemplateMetadata cursor : templates.getTemplates().values()) { + public static void toXContent(TemplatesMetadata templatesMetadata, XContentBuilder builder, Params params) throws IOException { + for (IndexTemplateMetadata cursor : templatesMetadata.getTemplates().values()) { IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params); } - // builder.endObject(); } public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException { Builder builder = new Builder(); - // we might get here after the templates-metadata element, or on a fresh parser XContentParser.Token token = parser.currentToken(); String currentFieldName = parser.currentName(); if (currentFieldName == null) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9d6cb4fcf48ef..bee6e98761ed6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -525,22 +525,16 @@ private UploadedMetadataResults writeMetadataInParallel( ) ); } - customToUpload.forEach( - (key, value) -> { - String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key); - uploadTasks.put( - customComponent, - getAsyncMetadataWriteAction( - clusterState, - customComponent, - CUSTOM_METADATA_FORMAT, - value, - listener - ) - ); - } - ); - indexToUpload.forEach(indexMetadata -> { uploadTasks.put(indexMetadata.getIndexName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); }); + customToUpload.forEach((key, value) -> { + String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key); + uploadTasks.put( + customComponent, + getAsyncMetadataWriteAction(clusterState, customComponent, CUSTOM_METADATA_FORMAT, value, listener) + ); + }); + indexToUpload.forEach(indexMetadata -> { + uploadTasks.put(indexMetadata.getIndexName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); + }); // start async upload of all required metadata files for (CheckedRunnable uploadTask : uploadTasks.values()) { @@ -554,7 +548,7 @@ private UploadedMetadataResults writeMetadataInParallel( String.format( Locale.ROOT, "Timed out waiting for transfer of following metadata to complete - %s", - String.join(", ", uploadTasks.keySet()) + String.join(", ", uploadTasks.keySet()) ) ); exceptionList.forEach(ex::addSuppressed); @@ -591,8 +585,10 @@ private UploadedMetadataResults writeMetadataInParallel( } else if (uploadedMetadata.getComponent().contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; - response.uploadedCustomMetadataMap.put(custom, - new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename())); + response.uploadedCustomMetadataMap.put( + custom, + new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename()) + ); } else if (COORDINATION_METADATA.equals(uploadedMetadata.getComponent())) { response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (SETTING_METADATA.equals(uploadedMetadata.getComponent())) { @@ -1091,8 +1087,13 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust builder.coordinationMetadata(coordinationMetadata); builder.persistentSettings(settingsMetadata); builder.templates(templatesMetadata); - clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> - builder.putCustom(key, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key))); + clusterMetadataManifest.getCustomMetadataMap() + .forEach( + (key, value) -> builder.putCustom( + key, + getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key) + ) + ); return builder.build(); } else { return Metadata.EMPTY_METADATA; @@ -1168,12 +1169,7 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste } } - private Metadata.Custom getCustomsMetadata( - String clusterName, - String clusterUUID, - String customMetadataFileName, - String custom - ) { + private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { requireNonNull(customMetadataFileName); try { // Fetch Custom metadata diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 5235dd456af7b..246be5fb34547 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -9,6 +9,9 @@ package org.opensearch.gateway.remote; import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexGraveyard; +import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; @@ -22,9 +25,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; public class ClusterMetadataManifestTests extends OpenSearchTestCase { @@ -100,7 +105,23 @@ public void testClusterMetadataManifestXContent() throws IOException { new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), - new HashMap<>() + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -130,7 +151,23 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), - new HashMap<>() + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) + ) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( From 0b3873655e5a18b20a98605a4473d7c0a3b02365 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 15 Mar 2024 14:07:01 +0530 Subject: [PATCH 008/133] Modify Custom's fromXContent Signed-off-by: Shivansh Arora --- .../java/org/opensearch/cluster/metadata/Metadata.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 222ecee5ad376..860b5738ff4ad 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -177,13 +177,8 @@ public interface Custom extends NamedDiffable, ToXContentFragment, Clust EnumSet context(); static Custom fromXContent(XContentParser parser, String name) throws IOException { - try { - return parser.namedObject(Custom.class, name, null); - } catch (NamedObjectNotFoundException e) { - logger.warn("Unknown custom object with type {}", name); - parser.skipChildren(); - throw e; - } + // handling any Exception is caller's responsibility + return parser.namedObject(Custom.class, name, null); } static Custom fromXContent(XContentParser parser) throws IOException { @@ -1849,6 +1844,7 @@ public static Metadata fromXContent(XContentParser parser) throws IOException { builder.putCustom(custom.getWriteableName(), custom); } catch (NamedObjectNotFoundException ex) { logger.warn("Skipping unknown custom object with type {}", currentFieldName); + parser.skipChildren(); } } } else if (token.isValue()) { From c86c0f1e7ec60de6ceb388b5937e5c7a32918bc9 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 18 Mar 2024 08:55:06 +0530 Subject: [PATCH 009/133] Remove ClusterMetadataManifest constructor Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 124 +++++++----------- .../RemoteClusterStateServiceTests.java | 42 ++---- 2 files changed, 57 insertions(+), 109 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index a1bba93477210..2671a91295a83 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -126,60 +126,58 @@ private static Map customMetadata(Object[] fi private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", - fields -> new ClusterMetadataManifest( - term(fields), - version(fields), - clusterUUID(fields), - stateUUID(fields), - opensearchVersion(fields), - nodeId(fields), - committed(fields), - CODEC_V0, - null, - indices(fields), - previousClusterUUID(fields), - clusterUUIDCommitted(fields) - ) + fields -> ClusterMetadataManifest.builder() + .clusterTerm(term(fields)) + .stateVersion(version(fields)) + .clusterUUID(clusterUUID(fields)) + .stateUUID(stateUUID(fields)) + .opensearchVersion(opensearchVersion(fields)) + .nodeId(nodeId(fields)) + .committed(committed(fields)) + .codecVersion(CODEC_V0) + .indices(indices(fields)) + .previousClusterUUID(previousClusterUUID(fields)) + .clusterUUIDCommitted(clusterUUIDCommitted(fields)) + .build() ); private static final ConstructingObjectParser PARSER_V1 = new ConstructingObjectParser<>( "cluster_metadata_manifest", - fields -> new ClusterMetadataManifest( - term(fields), - version(fields), - clusterUUID(fields), - stateUUID(fields), - opensearchVersion(fields), - nodeId(fields), - committed(fields), - codecVersion(fields), - globalMetadataFileName(fields), - indices(fields), - previousClusterUUID(fields), - clusterUUIDCommitted(fields) - ) + fields -> ClusterMetadataManifest.builder() + .clusterTerm(term(fields)) + .stateVersion(version(fields)) + .clusterUUID(clusterUUID(fields)) + .stateUUID(stateUUID(fields)) + .opensearchVersion(opensearchVersion(fields)) + .nodeId(nodeId(fields)) + .committed(committed(fields)) + .codecVersion(codecVersion(fields)) + .globalMetadataFileName(globalMetadataFileName(fields)) + .indices(indices(fields)) + .previousClusterUUID(previousClusterUUID(fields)) + .clusterUUIDCommitted(clusterUUIDCommitted(fields)) + .build() ); private static final ConstructingObjectParser PARSER_V2 = new ConstructingObjectParser<>( "cluster_metadata_manifest", - fields -> new ClusterMetadataManifest( - term(fields), - version(fields), - clusterUUID(fields), - stateUUID(fields), - opensearchVersion(fields), - nodeId(fields), - committed(fields), - codecVersion(fields), - null, - indices(fields), - previousClusterUUID(fields), - clusterUUIDCommitted(fields), - coordinationMetadata(fields), - settingsMetadata(fields), - templatesMetadata(fields), - customMetadata(fields) - ) + fields -> ClusterMetadataManifest.builder() + .clusterTerm(term(fields)) + .stateVersion(version(fields)) + .previousClusterUUID(clusterUUID(fields)) + .stateUUID(stateUUID(fields)) + .opensearchVersion(opensearchVersion(fields)) + .nodeId(nodeId(fields)) + .committed(committed(fields)) + .codecVersion(codecVersion(fields)) + .indices(indices(fields)) + .previousClusterUUID(previousClusterUUID(fields)) + .clusterUUIDCommitted(clusterUUIDCommitted(fields)) + .coordinationMetadata(coordinationMetadata(fields)) + .settingMetadata(settingsMetadata(fields)) + .templatesMetadata(templatesMetadata(fields)) + .customMetadataMap(customMetadata(fields)) + .build() ); private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; @@ -322,40 +320,6 @@ public boolean hasMetadataAttributesFiles() { || !uploadedCustomMetadataMap.isEmpty(); } - public ClusterMetadataManifest( - long clusterTerm, - long version, - String clusterUUID, - String stateUUID, - Version opensearchVersion, - String nodeId, - boolean committed, - int codecVersion, - String globalMetadataFileName, - List indices, - String previousClusterUUID, - boolean clusterUUIDCommitted - ) { - this( - clusterTerm, - version, - clusterUUID, - stateUUID, - opensearchVersion, - nodeId, - committed, - codecVersion, - globalMetadataFileName, - indices, - previousClusterUUID, - clusterUUIDCommitted, - null, - null, - null, - null - ); - } - public ClusterMetadataManifest( long clusterTerm, long version, diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index c3fd7098174f6..685c609c6e8ff 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -70,6 +70,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -250,28 +251,11 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { ArgumentCaptor> actionListenerArgumentCaptor = ArgumentCaptor.forClass(ActionListener.class); ArgumentCaptor writeContextArgumentCaptor = ArgumentCaptor.forClass(WriteContext.class); - AtomicReference capturedWriteContext = new AtomicReference<>(); + ConcurrentHashMap capturedWriteContext = new ConcurrentHashMap<>(); doAnswer((i) -> { actionListenerArgumentCaptor.getValue().onResponse(null); - return null; - }).doAnswer((i) -> { - actionListenerArgumentCaptor.getValue().onResponse(null); - return null; - }).doAnswer((i) -> { - actionListenerArgumentCaptor.getValue().onResponse(null); - return null; - }).doAnswer((i) -> { - actionListenerArgumentCaptor.getValue().onResponse(null); - return null; - }).doAnswer((i) -> { - actionListenerArgumentCaptor.getValue().onResponse(null); - return null; - }).doAnswer((i) -> { - actionListenerArgumentCaptor.getValue().onResponse(null); - capturedWriteContext.set(writeContextArgumentCaptor.getValue()); - return null; - }).doAnswer((i) -> { - actionListenerArgumentCaptor.getValue().onResponse(null); + WriteContext writeContext = writeContextArgumentCaptor.getValue(); + capturedWriteContext.put(writeContext.getFileName().split(DELIMITER)[0], writeContextArgumentCaptor.getValue()); return null; }).when(container).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture()); @@ -308,27 +292,27 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { assertEquals(7, actionListenerArgumentCaptor.getAllValues().size()); assertEquals(7, writeContextArgumentCaptor.getAllValues().size()); - byte[] writtenBytes = capturedWriteContext.get() + byte[] writtenBytes = capturedWriteContext.get("metadata") .getStreamProvider(Integer.MAX_VALUE) .provideStream(0) .getInputStream() .readAllBytes(); IndexMetadata writtenIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.deserialize( - capturedWriteContext.get().getFileName(), + capturedWriteContext.get("metadata").getFileName(), blobStoreRepository.getNamedXContentRegistry(), new BytesArray(writtenBytes) ); - assertEquals(capturedWriteContext.get().getWritePriority(), WritePriority.URGENT); + assertEquals(capturedWriteContext.get("metadata").getWritePriority(), WritePriority.URGENT); assertEquals(writtenIndexMetadata.getNumberOfShards(), 1); assertEquals(writtenIndexMetadata.getNumberOfReplicas(), 0); assertEquals(writtenIndexMetadata.getIndex().getName(), "test-index"); assertEquals(writtenIndexMetadata.getIndex().getUUID(), "index-uuid"); long expectedChecksum = RemoteTransferContainer.checksumOfChecksum(new ByteArrayIndexInput("metadata-filename", writtenBytes), 8); - if (capturedWriteContext.get().doRemoteDataIntegrityCheck()) { - assertEquals(capturedWriteContext.get().getExpectedChecksum().longValue(), expectedChecksum); + if (capturedWriteContext.get("metadata").doRemoteDataIntegrityCheck()) { + assertEquals(capturedWriteContext.get("metadata").getExpectedChecksum().longValue(), expectedChecksum); } else { - assertEquals(capturedWriteContext.get().getExpectedChecksum(), null); + assertEquals(capturedWriteContext.get("metadata").getExpectedChecksum(), null); } } @@ -386,7 +370,7 @@ public void testTimeoutWhileWritingManifestFile() throws IOException { remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)); } catch (Exception e) { assertTrue(e instanceof RemoteClusterStateService.RemoteStateTransferException); - assertTrue(e.getMessage().contains("Timed out waiting for transfer of metadata to complete")); + assertTrue(e.getMessage().contains("Timed out waiting for transfer of following metadata to complete")); } } @@ -967,7 +951,7 @@ public void testReadGlobalMetadata() throws IOException { .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) - .put(IndexGraveyard.TYPE, new UploadedMetadataAttribute(CUSTOM_METADATA + CUSTOM_DELIMITER + IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) + .put(IndexGraveyard.TYPE, new UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") @@ -976,7 +960,7 @@ public void testReadGlobalMetadata() throws IOException { Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build(); mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expactedMetadata); - ClusterState newClusterState = remoteClusterStateService.getLatestClusterState( + ClusterState newClusterState = remoteClusterStateService.getLatestClusterState( clusterState.getClusterName().value(), clusterState.metadata().clusterUUID() ); From 3ed92e5f458585017a77235cd60c461aacb8345b Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 18 Mar 2024 16:50:31 +0530 Subject: [PATCH 010/133] Added tests Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 2 + .../RemoteStoreClusterStateRestoreIT.java | 83 ++++++++++++++++++- .../cluster/metadata/TemplatesMetadata.java | 6 +- .../remote/ClusterMetadataManifest.java | 2 +- .../remote/RemoteClusterStateService.java | 39 +++++---- .../coordination/CoordinationStateTests.java | 28 +++---- .../remote/ClusterMetadataManifestTests.java | 58 ++++++------- .../RemoteClusterStateServiceTests.java | 49 ++++++++++- 8 files changed, 197 insertions(+), 70 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index 61b34af5be3ba..d4c4bada36b36 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -10,6 +10,8 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.opensearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.opensearch.action.update.UpdateRequest; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c61e2ec6e4f6c..322a440f97e9e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -8,16 +8,31 @@ package org.opensearch.remotestore; +import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; +import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; +import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.datastream.DataStreamRolloverIT; import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.metadata.ComponentTemplate; +import org.opensearch.cluster.metadata.ComponentTemplateMetadata; +import org.opensearch.cluster.metadata.ComposableIndexTemplate; +import org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.Template; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; @@ -25,15 +40,18 @@ import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; +import javax.swing.*; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; @@ -46,6 +64,11 @@ @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreClusterStateRestoreIT extends BaseRemoteStoreRestoreIT { + static final String TEMPLATE_NAME = "remote-store-test-template"; + static final String COMPONENT_TEMPLATE_NAME = "remote-component-template1"; + static final String COMPOSABLE_TEMPLATE_NAME = "remote-composable-template1"; + static final Setting MOCK_SETTING = Setting.simpleString("mock-setting"); + static final String[] EXCLUDED_NODES = {"ex-1", "ex-2"}; @Override protected Settings nodeSettings(int nodeOrdinal) { @@ -87,6 +110,50 @@ public void testFullClusterRestore() throws Exception { Map indexStats = initialTestSetup(shardCount, replicaCount, dataNodeCount, 1); String prevClusterUUID = clusterService().state().metadata().clusterUUID(); long prevClusterStateVersion = clusterService().state().version(); + // Step - 1.1 Add some cluster state elements + ActionFuture response = client().admin().indices().preparePutTemplate(TEMPLATE_NAME) + .addAlias(new Alias(INDEX_NAME)) + .setPatterns(Arrays.stream(INDEX_NAMES_WILDCARD.split(",")).collect(Collectors.toList())) + .execute(); + assertTrue(response.get().isAcknowledged()); + ActionFuture clusterUpdateSettingsResponse = client().admin().cluster() + .prepareUpdateSettings().setPersistentSettings( + Settings.builder().put(SETTING_READ_ONLY_SETTING.getKey(), false).build() + ).execute(); + assertTrue(clusterUpdateSettingsResponse.get().isAcknowledged()); + // update coordination metadata + client().execute( + AddVotingConfigExclusionsAction.INSTANCE, + new AddVotingConfigExclusionsRequest(EXCLUDED_NODES) + ); + // Add a custom metadata as component index template + ActionFuture componentTemplateResponse = client().execute( + PutComponentTemplateAction.INSTANCE, + new PutComponentTemplateAction.Request(COMPONENT_TEMPLATE_NAME) + .componentTemplate( + new ComponentTemplate(new Template(Settings.EMPTY, null, Collections.emptyMap()), + 1L, + Collections.emptyMap() + ) + ) + ); + assertTrue(componentTemplateResponse.get().isAcknowledged()); + ActionFuture composableTemplateResponse = client().execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request(COMPOSABLE_TEMPLATE_NAME) + .indexTemplate( + new ComposableIndexTemplate( + Arrays.stream(INDEX_NAMES_WILDCARD.split(",")).collect(Collectors.toList()), + new Template(Settings.EMPTY, null, Collections.emptyMap()), + Collections.singletonList(COMPONENT_TEMPLATE_NAME), + 1L, + 1L, + Collections.emptyMap(), + null + ) + ) + ); + assertTrue(composableTemplateResponse.get().isAcknowledged()); // Step - 2 Replace all nodes in the cluster with new nodes. This ensures new cluster state doesn't have previous index metadata resetCluster(dataNodeCount, clusterManagerNodeCount); @@ -104,7 +171,21 @@ public void testFullClusterRestore() throws Exception { ); validateMetadata(List.of(INDEX_NAME)); verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true); - + clusterService().state().metadata().coordinationMetadata().getVotingConfigExclusions().stream().forEach(config -> + assertTrue(Arrays.stream(EXCLUDED_NODES).anyMatch(node -> node.equals(config.getNodeId()))) + ); + assertFalse(clusterService().state().metadata().templates().isEmpty()); + assertTrue(clusterService().state().metadata().templates().containsKey(TEMPLATE_NAME)); + assertFalse(clusterService().state().metadata().settings().isEmpty()); + assertFalse(clusterService().state().metadata().settings().getAsBoolean(SETTING_READ_ONLY_SETTING.getKey(), true)); + assertNotNull(clusterService().state().metadata().custom("component_template")); + ComponentTemplateMetadata componentTemplateMetadata = clusterService().state().metadata().custom("component_template"); + assertFalse(componentTemplateMetadata.componentTemplates().isEmpty()); + assertTrue(componentTemplateMetadata.componentTemplates().containsKey(COMPONENT_TEMPLATE_NAME)); + assertNotNull(clusterService().state().metadata().custom("index_template")); + ComposableIndexTemplateMetadata composableIndexTemplate = clusterService().state().metadata().custom("index_template"); + assertFalse(composableIndexTemplate.indexTemplates().isEmpty()); + assertTrue(composableIndexTemplate.indexTemplates().containsKey(COMPOSABLE_TEMPLATE_NAME)); } /** diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 36205be4d7e08..84c62c22799e8 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -133,14 +133,10 @@ public static TemplatesMetadata fromXContent(XContentParser parser) throws IOExc if (token == XContentParser.Token.START_OBJECT) { // move to the field name token = parser.nextToken(); - if (token == XContentParser.Token.FIELD_NAME) { - // move to the next object - token = parser.nextToken(); - } } currentFieldName = parser.currentName(); } - if (currentFieldName != null && token == XContentParser.Token.START_OBJECT) { + if (currentFieldName != null) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName())); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 2671a91295a83..023eb06b8eb86 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -164,7 +164,7 @@ private static Map customMetadata(Object[] fi fields -> ClusterMetadataManifest.builder() .clusterTerm(term(fields)) .stateVersion(version(fields)) - .previousClusterUUID(clusterUUID(fields)) + .clusterUUID(clusterUUID(fields)) .stateUUID(stateUUID(fields)) .opensearchVersion(opensearchVersion(fields)) .nodeId(nodeId(fields)) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index bee6e98761ed6..19f526dcd65dd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -348,6 +348,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( final Map customsToUpload = getUpdatedCustoms(clusterState, previousClusterState); final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); for (final String custom : clusterState.metadata().customs().keySet()) { + // remove all the customs which are present currently previousStateCustomMap.remove(custom); } @@ -398,14 +399,14 @@ public ClusterMetadataManifest writeIncrementalMetadata( ); } + // update the map if the metadata was uploaded uploadedMetadataResults.uploadedIndexMetadata.forEach( uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata) ); allUploadedCustomMap.putAll(uploadedMetadataResults.uploadedCustomMetadataMap); + // remove the data for removed custom/indices previousStateCustomMap.keySet().forEach(allUploadedCustomMap::remove); - for (String removedIndexName : previousStateIndexMetadataVersionByName.keySet()) { - allUploadedIndexMetadata.remove(removedIndexName); - } + previousStateIndexMetadataVersionByName.keySet().forEach(allUploadedIndexMetadata::remove); final ClusterMetadataManifest manifest = uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), @@ -726,20 +727,20 @@ private ClusterMetadataManifest uploadV1Manifest( committed, ClusterMetadataManifest.CODEC_V1 ); - final ClusterMetadataManifest manifest = new ClusterMetadataManifest( - clusterState.term(), - clusterState.getVersion(), - clusterState.metadata().clusterUUID(), - clusterState.stateUUID(), - Version.CURRENT, - nodeId, - committed, - ClusterMetadataManifest.CODEC_V1, - globalMetadataFileName, - uploadedIndexMetadata, - previousClusterUUID, - clusterState.metadata().clusterUUIDCommitted() - ); + ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(clusterState.term()) + .stateVersion(clusterState.getVersion()) + .clusterUUID(clusterState.metadata().clusterUUID()) + .stateUUID(clusterState.stateUUID()) + .opensearchVersion(Version.CURRENT) + .nodeId(nodeId) + .committed(committed) + .codecVersion(ClusterMetadataManifest.CODEC_V1) + .globalMetadataFileName(globalMetadataFileName) + .indices(uploadedIndexMetadata) + .previousClusterUUID(previousClusterUUID) + .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) + .build(); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; } @@ -897,7 +898,9 @@ private Map getUpdatedCustoms(ClusterState currentState Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { - if (!cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { + if (currentCustoms.contains(cursor.getKey()) && + !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { + // If the custom metadata is updated, we need to upload the new version. updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); } currentCustoms.remove(cursor.getKey()); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 1c0dc7fc1ca2d..bd71aecf89101 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -930,20 +930,20 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep final VotingConfiguration initialConfig = VotingConfiguration.of(node1); final ClusterState clusterState = clusterState(0L, 0L, node1, initialConfig, initialConfig, 42L); final String previousClusterUUID = "prev-cluster-uuid"; - final ClusterMetadataManifest manifest = new ClusterMetadataManifest( - 0L, - 0L, - randomAlphaOfLength(10), - randomAlphaOfLength(10), - Version.CURRENT, - randomAlphaOfLength(10), - false, - 1, - randomAlphaOfLength(10), - Collections.emptyList(), - randomAlphaOfLength(10), - true - ); + final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(0L) + .stateVersion(0L) + .clusterUUID(randomAlphaOfLength(10)) + .stateUUID(randomAlphaOfLength(10)) + .opensearchVersion(Version.CURRENT) + .nodeId(randomAlphaOfLength(10)) + .committed(false) + .codecVersion(1) + .globalMetadataFileName(randomAlphaOfLength(10)) + .indices(Collections.emptyList()) + .previousClusterUUID(randomAlphaOfLength(10)) + .clusterUUIDCommitted(true) + .build(); Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 246be5fb34547..0b3cd49140939 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -31,24 +31,26 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; + public class ClusterMetadataManifestTests extends OpenSearchTestCase { public void testClusterMetadataManifestXContentV0() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V0, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true - ); + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(CODEC_V0) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -62,20 +64,20 @@ public void testClusterMetadataManifestXContentV0() throws IOException { public void testClusterMetadataManifestXContentV1() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V1, - "test-global-metadata-file", - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true - ); + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(CODEC_V1) + .globalMetadataFileName("test-global-metadata-file") + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 685c609c6e8ff..a6085ff8fbd3a 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -727,6 +727,52 @@ public void testCustomMetadataOnlyUpdated() throws IOException { }); } + public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException { + // setup + mockBlobStoreObjects(); + + // Initial cluster state with index. + final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_"); + + ClusterState clusterState1 = ClusterState.builder(initialClusterState) + .metadata( + Metadata.builder(initialClusterState.metadata()) + .putCustom("custom1", new CustomMetadata1("mock_custom_metadata1")) + .putCustom("custom2", new CustomMetadata1("mock_custom_metadata2")) + .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3")) + ).build(); + + ClusterMetadataManifest manifest1 = remoteClusterStateService.writeIncrementalMetadata( + initialClusterState, + clusterState1, + initialManifest + ); + // remove custom1 from the cluster state, update custom2, custom3 is at it is, added custom4 + ClusterState clusterState2 = ClusterState.builder(initialClusterState) + .metadata( + Metadata.builder(initialClusterState.metadata()) + .putCustom("custom2", new CustomMetadata1("mock_updated_custom_metadata")) + .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3")) + .putCustom("custom4", new CustomMetadata1("mock_custom_metadata4")) + ).build(); + ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata( + clusterState1, + clusterState2, + manifest1 + ); + // custom1 is removed + assertFalse(manifest2.getCustomMetadataMap().containsKey("custom1")); + // custom2 is updated + assertNotEquals(manifest1.getCustomMetadataMap().get("custom2"), manifest2.getCustomMetadataMap().get("custom2")); + // custom3 is unchanged + assertEquals(manifest1.getCustomMetadataMap().get("custom3"), manifest2.getCustomMetadataMap().get("custom3")); + // custom4 is added + assertTrue(manifest2.getCustomMetadataMap().containsKey("custom4")); + assertFalse(manifest1.getCustomMetadataMap().containsKey("custom4")); + } + /* * Here we will verify global metadata is not uploaded again if change is only in index metadata */ @@ -758,9 +804,6 @@ public void testIndexMetadataOnlyUpdated() throws IOException { }); } - /* - * Here we will verify index metadata is not uploaded again if change is only in global metadata - */ private void verifyMetadataAttributeOnlyUpdated( Function clusterStateUpdater, BiConsumer assertions From e1f517e7a03ad5cb4694aa0d7b6ba185d4f46715 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 19 Mar 2024 11:45:13 +0530 Subject: [PATCH 011/133] remove stale global metadata files Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateService.java | 66 ++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 19f526dcd65dd..bef3d764a9aeb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1583,7 +1583,16 @@ private void deleteClusterMetadata( ); clusterMetadataManifest.getIndices() .forEach(uploadedIndexMetadata -> filesToKeep.add(uploadedIndexMetadata.getUploadedFilename())); - filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName()); + if (clusterMetadataManifest.getGlobalMetadataFileName() != null) { + filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName()); + } else { + filesToKeep.add(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); + filesToKeep.add(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); + filesToKeep.add(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); + clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> { + filesToKeep.add(value.getUploadedFilename()); + }); + } }); staleManifestBlobMetadata.forEach(blobMetadata -> { ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest( @@ -1592,14 +1601,55 @@ private void deleteClusterMetadata( blobMetadata.name() ); staleManifestPaths.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blobMetadata.name()); - if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) { - String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); - staleGlobalMetadataPaths.add( - new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( - globalMetadataSplitPath[globalMetadataSplitPath.length - 1] - ) - ); + if (clusterMetadataManifest.getGlobalMetadataFileName() != null) { + if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) { + String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); + staleGlobalMetadataPaths.add( + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( + globalMetadataSplitPath[globalMetadataSplitPath.length - 1] + ) + ); + } + } else { + if (filesToKeep.contains(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()) == false) { + String[] coordinationMetadataSplitPath = clusterMetadataManifest.getCoordinationMetadata() + .getUploadedFilename().split("/"); + staleGlobalMetadataPaths.add( + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( + coordinationMetadataSplitPath[coordinationMetadataSplitPath.length - 1] + ) + ); + } + if (filesToKeep.contains(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()) == false) { + String[] templatesMetadataSplitPath = clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() + .split("/"); + staleGlobalMetadataPaths.add( + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( + templatesMetadataSplitPath[templatesMetadataSplitPath.length - 1] + ) + ); + } + if (filesToKeep.contains(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()) == false) { + String[] settingsMetadataSplitPath = clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() + .split("/"); + staleGlobalMetadataPaths.add( + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( + settingsMetadataSplitPath[settingsMetadataSplitPath.length - 1] + ) + ); + } + clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> { + if (filesToKeep.contains(value.getUploadedFilename()) == false) { + String[] customMetadataSplitPath = value.getUploadedFilename().split("/"); + staleGlobalMetadataPaths.add( + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( + customMetadataSplitPath[customMetadataSplitPath.length - 1] + ) + ); + } + }); } + clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) { staleIndexMetadataPaths.add( From cd5c9a5b256119fe2dbe71c6832343e20d3b9ee6 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 19 Mar 2024 11:57:52 +0530 Subject: [PATCH 012/133] spotless apply Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 2 - .../RemoteStoreClusterStateRestoreIT.java | 62 +++++++++---------- .../remote/RemoteClusterStateService.java | 18 +++--- .../RemoteClusterStateServiceTests.java | 15 ++--- 4 files changed, 43 insertions(+), 54 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index d4c4bada36b36..61b34af5be3ba 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -10,8 +10,6 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.opensearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.opensearch.action.update.UpdateRequest; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index 322a440f97e9e..977bee25bd621 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -10,7 +10,6 @@ import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction; import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest; -import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.action.admin.indices.alias.Alias; @@ -40,7 +39,6 @@ import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; -import javax.swing.*; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -68,7 +66,7 @@ public class RemoteStoreClusterStateRestoreIT extends BaseRemoteStoreRestoreIT { static final String COMPONENT_TEMPLATE_NAME = "remote-component-template1"; static final String COMPOSABLE_TEMPLATE_NAME = "remote-composable-template1"; static final Setting MOCK_SETTING = Setting.simpleString("mock-setting"); - static final String[] EXCLUDED_NODES = {"ex-1", "ex-2"}; + static final String[] EXCLUDED_NODES = { "ex-1", "ex-2" }; @Override protected Settings nodeSettings(int nodeOrdinal) { @@ -111,47 +109,42 @@ public void testFullClusterRestore() throws Exception { String prevClusterUUID = clusterService().state().metadata().clusterUUID(); long prevClusterStateVersion = clusterService().state().version(); // Step - 1.1 Add some cluster state elements - ActionFuture response = client().admin().indices().preparePutTemplate(TEMPLATE_NAME) + ActionFuture response = client().admin() + .indices() + .preparePutTemplate(TEMPLATE_NAME) .addAlias(new Alias(INDEX_NAME)) .setPatterns(Arrays.stream(INDEX_NAMES_WILDCARD.split(",")).collect(Collectors.toList())) .execute(); assertTrue(response.get().isAcknowledged()); - ActionFuture clusterUpdateSettingsResponse = client().admin().cluster() - .prepareUpdateSettings().setPersistentSettings( - Settings.builder().put(SETTING_READ_ONLY_SETTING.getKey(), false).build() - ).execute(); + ActionFuture clusterUpdateSettingsResponse = client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(SETTING_READ_ONLY_SETTING.getKey(), false).build()) + .execute(); assertTrue(clusterUpdateSettingsResponse.get().isAcknowledged()); // update coordination metadata - client().execute( - AddVotingConfigExclusionsAction.INSTANCE, - new AddVotingConfigExclusionsRequest(EXCLUDED_NODES) - ); + client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(EXCLUDED_NODES)); // Add a custom metadata as component index template ActionFuture componentTemplateResponse = client().execute( PutComponentTemplateAction.INSTANCE, - new PutComponentTemplateAction.Request(COMPONENT_TEMPLATE_NAME) - .componentTemplate( - new ComponentTemplate(new Template(Settings.EMPTY, null, Collections.emptyMap()), - 1L, - Collections.emptyMap() - ) - ) + new PutComponentTemplateAction.Request(COMPONENT_TEMPLATE_NAME).componentTemplate( + new ComponentTemplate(new Template(Settings.EMPTY, null, Collections.emptyMap()), 1L, Collections.emptyMap()) + ) ); assertTrue(componentTemplateResponse.get().isAcknowledged()); ActionFuture composableTemplateResponse = client().execute( PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request(COMPOSABLE_TEMPLATE_NAME) - .indexTemplate( - new ComposableIndexTemplate( - Arrays.stream(INDEX_NAMES_WILDCARD.split(",")).collect(Collectors.toList()), - new Template(Settings.EMPTY, null, Collections.emptyMap()), - Collections.singletonList(COMPONENT_TEMPLATE_NAME), - 1L, - 1L, - Collections.emptyMap(), - null - ) + new PutComposableIndexTemplateAction.Request(COMPOSABLE_TEMPLATE_NAME).indexTemplate( + new ComposableIndexTemplate( + Arrays.stream(INDEX_NAMES_WILDCARD.split(",")).collect(Collectors.toList()), + new Template(Settings.EMPTY, null, Collections.emptyMap()), + Collections.singletonList(COMPONENT_TEMPLATE_NAME), + 1L, + 1L, + Collections.emptyMap(), + null ) + ) ); assertTrue(composableTemplateResponse.get().isAcknowledged()); @@ -171,9 +164,12 @@ public void testFullClusterRestore() throws Exception { ); validateMetadata(List.of(INDEX_NAME)); verifyRedIndicesAndTriggerRestore(indexStats, INDEX_NAME, true); - clusterService().state().metadata().coordinationMetadata().getVotingConfigExclusions().stream().forEach(config -> - assertTrue(Arrays.stream(EXCLUDED_NODES).anyMatch(node -> node.equals(config.getNodeId()))) - ); + clusterService().state() + .metadata() + .coordinationMetadata() + .getVotingConfigExclusions() + .stream() + .forEach(config -> assertTrue(Arrays.stream(EXCLUDED_NODES).anyMatch(node -> node.equals(config.getNodeId())))); assertFalse(clusterService().state().metadata().templates().isEmpty()); assertTrue(clusterService().state().metadata().templates().containsKey(TEMPLATE_NAME)); assertFalse(clusterService().state().metadata().settings().isEmpty()); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index bef3d764a9aeb..ac9a14d0ba74f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -898,8 +898,8 @@ private Map getUpdatedCustoms(ClusterState currentState Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { - if (currentCustoms.contains(cursor.getKey()) && - !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { + if (currentCustoms.contains(cursor.getKey()) + && !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { // If the custom metadata is updated, we need to upload the new version. updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); } @@ -1589,9 +1589,8 @@ private void deleteClusterMetadata( filesToKeep.add(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); filesToKeep.add(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); filesToKeep.add(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); - clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) -> { - filesToKeep.add(value.getUploadedFilename()); - }); + clusterMetadataManifest.getCustomMetadataMap() + .forEach((key, value) -> { filesToKeep.add(value.getUploadedFilename()); }); } }); staleManifestBlobMetadata.forEach(blobMetadata -> { @@ -1613,7 +1612,8 @@ private void deleteClusterMetadata( } else { if (filesToKeep.contains(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()) == false) { String[] coordinationMetadataSplitPath = clusterMetadataManifest.getCoordinationMetadata() - .getUploadedFilename().split("/"); + .getUploadedFilename() + .split("/"); staleGlobalMetadataPaths.add( new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( coordinationMetadataSplitPath[coordinationMetadataSplitPath.length - 1] @@ -1621,7 +1621,8 @@ private void deleteClusterMetadata( ); } if (filesToKeep.contains(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()) == false) { - String[] templatesMetadataSplitPath = clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() + String[] templatesMetadataSplitPath = clusterMetadataManifest.getTemplatesMetadata() + .getUploadedFilename() .split("/"); staleGlobalMetadataPaths.add( new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( @@ -1630,8 +1631,7 @@ private void deleteClusterMetadata( ); } if (filesToKeep.contains(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()) == false) { - String[] settingsMetadataSplitPath = clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() - .split("/"); + String[] settingsMetadataSplitPath = clusterMetadataManifest.getSettingsMetadata().getUploadedFilename().split("/"); staleGlobalMetadataPaths.add( new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( settingsMetadataSplitPath[settingsMetadataSplitPath.length - 1] diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index a6085ff8fbd3a..e4b41d30ba677 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -74,7 +74,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; @@ -87,8 +86,6 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; @@ -742,7 +739,8 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException { .putCustom("custom1", new CustomMetadata1("mock_custom_metadata1")) .putCustom("custom2", new CustomMetadata1("mock_custom_metadata2")) .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3")) - ).build(); + ) + .build(); ClusterMetadataManifest manifest1 = remoteClusterStateService.writeIncrementalMetadata( initialClusterState, @@ -756,12 +754,9 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException { .putCustom("custom2", new CustomMetadata1("mock_updated_custom_metadata")) .putCustom("custom3", new CustomMetadata1("mock_custom_metadata3")) .putCustom("custom4", new CustomMetadata1("mock_custom_metadata4")) - ).build(); - ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata( - clusterState1, - clusterState2, - manifest1 - ); + ) + .build(); + ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1); // custom1 is removed assertFalse(manifest2.getCustomMetadataMap().containsKey("custom1")); // custom2 is updated From 33cfeb0239f504934964569fb8813b21b062e8ef Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 3 Apr 2024 14:13:40 +0530 Subject: [PATCH 013/133] Refactored RemoteClusterStateService into smaller files Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 12 +- .../RemoteStoreClusterStateRestoreIT.java | 7 +- .../cluster/metadata/TemplatesMetadata.java | 3 + .../common/settings/ClusterSettings.java | 9 +- .../remote/RemoteClusterStateService.java | 930 ++---------------- .../remote/RemoteClusterStateUtils.java | 99 ++ .../remote/RemoteGlobalMetadataManager.java | 334 +++++++ .../remote/RemoteIndexMetadataManager.java | 182 ++++ .../gateway/remote/RemoteManifestManager.java | 357 +++++++ .../remote/ClusterMetadataManifestTests.java | 26 +- .../RemoteClusterStateServiceTests.java | 190 +--- .../RemoteGlobalMetadataManagerTests.java | 52 + .../RemoteIndexMetadataManagerTests.java | 52 + .../remote/RemoteManifestManagerTests.java | 143 +++ 14 files changed, 1384 insertions(+), 1012 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index 61b34af5be3ba..4fc6550f2a3a6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -27,13 +27,13 @@ import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; -import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index 8c8209b80bfd8..34d223f1dd14f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -36,6 +36,7 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; @@ -320,9 +321,7 @@ public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathTh // Step - 3 Delete index metadata file in remote try { Files.move( - segmentRepoPath.resolve( - RemoteClusterStateService.encodeString(clusterName) + "/cluster-state/" + prevClusterUUID + "/index" - ), + segmentRepoPath.resolve(RemoteClusterStateUtils.encodeString(clusterName) + "/cluster-state/" + prevClusterUUID + "/index"), segmentRepoPath.resolve("cluster-state/") ); } catch (IOException e) { @@ -348,7 +347,7 @@ public void testRemoteStateFullRestart() throws Exception { try { Files.move( segmentRepoPath.resolve( - RemoteClusterStateService.encodeString(clusterService().state().getClusterName().value()) + RemoteClusterStateUtils.encodeString(clusterService().state().getClusterName().value()) + "/cluster-state/" + prevClusterUUID + "/manifest" diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 84c62c22799e8..2ffcd7fb6988e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -9,6 +9,7 @@ package org.opensearch.cluster.metadata; import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -25,6 +26,7 @@ * * @opensearch.internal */ +@PublicApi(since = "1.0.0") public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment { public static TemplatesMetadata EMPTY_METADATA = builder().build(); private final Map templates; @@ -83,6 +85,7 @@ public int hashCode() { * * @opensearch.api */ + @PublicApi(since = "1.0.0") public static class Builder { private final Map templates; diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 9d763c970c3e7..81ee55fcc31d4 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -102,6 +102,9 @@ import org.opensearch.gateway.GatewayService; import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.RemoteGlobalMetadataManager; +import org.opensearch.gateway.remote.RemoteIndexMetadataManager; +import org.opensearch.gateway.remote.RemoteManifestManager; import org.opensearch.http.HttpTransportSettings; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; @@ -703,9 +706,9 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, - RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, - RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, - RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, + RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, + RemoteGlobalMetadataManager.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, + RemoteManifestManager.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING, RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING, IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ac9a14d0ba74f..1f7f40b8fb598 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -11,13 +11,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.Version; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; @@ -30,44 +27,55 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; -import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Supplier; import java.util.stream.Collectors; -import static java.util.Objects.requireNonNull; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -76,102 +84,10 @@ * @opensearch.internal */ public class RemoteClusterStateService implements Closeable { - - public static final String METADATA_NAME_FORMAT = "%s.dat"; - - public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int RETAINED_MANIFESTS = 10; - public static final String DELIMITER = "__"; - public static final String CUSTOM_DELIMITER = "--"; - private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); - public static final TimeValue INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); - - public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); - - public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); - - public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( - "cluster.remote_store.state.index_metadata.upload_timeout", - INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT, - Setting.Property.Dynamic, - Setting.Property.NodeScope - ); - - public static final Setting GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( - "cluster.remote_store.state.global_metadata.upload_timeout", - GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT, - Setting.Property.Dynamic, - Setting.Property.NodeScope - ); - - public static final Setting METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( - "cluster.remote_store.state.metadata_manifest.upload_timeout", - METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT, - Setting.Property.Dynamic, - Setting.Property.NodeScope - ); - - public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "index-metadata", - METADATA_NAME_FORMAT, - IndexMetadata::fromXContent - ); - - public static final ChecksumBlobStoreFormat GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "metadata", - METADATA_NAME_FORMAT, - Metadata::fromXContent - ); - - public static final ChecksumBlobStoreFormat COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "coordination", - METADATA_NAME_FORMAT, - CoordinationMetadata::fromXContent - ); - - public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "settings", - METADATA_NAME_FORMAT, - Settings::fromXContent - ); - - public static final ChecksumBlobStoreFormat TEMPLATES_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "templates", - METADATA_NAME_FORMAT, - TemplatesMetadata::fromXContent - ); - - public static final ChecksumBlobStoreFormat CUSTOM_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "custom", - METADATA_NAME_FORMAT, - Metadata.Custom::fromXContent - ); - - /** - * Manifest format compatible with older codec v0, where codec version was missing. - */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V0 = - new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); - - /** - * Manifest format compatible with older codec v1, where global metadata was missing. - */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = - new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); - - /** - * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. - */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( - "cluster-metadata-manifest", - METADATA_MANIFEST_NAME_FORMAT, - ClusterMetadataManifest::fromXContent - ); - /** * Used to specify if cluster state metadata should be published to remote store */ @@ -182,18 +98,6 @@ public class RemoteClusterStateService implements Closeable { Property.Final ); - public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; - public static final String INDEX_PATH_TOKEN = "index"; - public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; - public static final String MANIFEST_PATH_TOKEN = "manifest"; - public static final String MANIFEST_FILE_PREFIX = "manifest"; - public static final String METADATA_FILE_PREFIX = "metadata"; - public static final String COORDINATION_METADATA = "coordination"; - public static final String SETTING_METADATA = "settings"; - public static final String TEMPLATES_METADATA = "templates"; - public static final String CUSTOM_METADATA = "custom"; - public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion - private final String nodeId; private final Supplier repositoriesService; private final Settings settings; @@ -203,24 +107,13 @@ public class RemoteClusterStateService implements Closeable { private BlobStoreTransferService blobStoreTransferService; private volatile TimeValue slowWriteLoggingThreshold; - private volatile TimeValue indexMetadataUploadTimeout; - private volatile TimeValue globalMetadataUploadTimeout; - private volatile TimeValue metadataManifestUploadTimeout; - private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; + private RemoteIndexMetadataManager remoteIndexMetadataManager; + private RemoteGlobalMetadataManager remoteGlobalMetadataManager; + private RemoteManifestManager remoteManifestManager; + private ClusterSettings clusterSettings; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; - public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; - - // ToXContent Params with gateway mode. - // We are using gateway context mode to persist all custom metadata. - public static final ToXContent.Params FORMAT_PARAMS; - static { - Map params = new HashMap<>(1); - params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY); - FORMAT_PARAMS = new ToXContent.MapParams(params); - } public RemoteClusterStateService( String nodeId, @@ -236,14 +129,9 @@ public RemoteClusterStateService( this.settings = settings; this.relativeTimeNanosSupplier = relativeTimeNanosSupplier; this.threadpool = threadPool; + this.clusterSettings = clusterSettings; this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); - this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); - this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); - this.metadataManifestUploadTimeout = clusterSettings.get(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); - clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); - clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); - clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); } @@ -276,7 +164,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true, true ); - final ClusterMetadataManifest manifest = uploadManifest( + final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, uploadedMetadataResults.uploadedIndexMetadata, previousClusterUUID, @@ -345,7 +233,10 @@ public ClusterMetadataManifest writeIncrementalMetadata( clusterState.metadata() ) == false; final Map previousStateCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); - final Map customsToUpload = getUpdatedCustoms(clusterState, previousClusterState); + final Map customsToUpload = remoteGlobalMetadataManager.getUpdatedCustoms( + clusterState, + previousClusterState + ); final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); for (final String custom : clusterState.metadata().customs().keySet()) { // remove all the customs which are present currently @@ -407,7 +298,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( // remove the data for removed custom/indices previousStateCustomMap.keySet().forEach(allUploadedCustomMap::remove); previousStateIndexMetadataVersionByName.keySet().forEach(allUploadedIndexMetadata::remove); - final ClusterMetadataManifest manifest = uploadManifest( + final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), previousManifest.getPreviousClusterUUID(), @@ -493,7 +384,7 @@ private UploadedMetadataResults writeMetadataInParallel( if (uploadSettingsMetadata) { uploadTasks.put( SETTING_METADATA, - getAsyncMetadataWriteAction( + remoteGlobalMetadataManager.getAsyncMetadataWriteAction( clusterState, SETTING_METADATA, SETTINGS_METADATA_FORMAT, @@ -505,7 +396,7 @@ private UploadedMetadataResults writeMetadataInParallel( if (uploadCoordinationMetadata) { uploadTasks.put( COORDINATION_METADATA, - getAsyncMetadataWriteAction( + remoteGlobalMetadataManager.getAsyncMetadataWriteAction( clusterState, COORDINATION_METADATA, COORDINATION_METADATA_FORMAT, @@ -517,7 +408,7 @@ private UploadedMetadataResults writeMetadataInParallel( if (uploadTemplateMetadata) { uploadTasks.put( TEMPLATES_METADATA, - getAsyncMetadataWriteAction( + remoteGlobalMetadataManager.getAsyncMetadataWriteAction( clusterState, TEMPLATES_METADATA, TEMPLATES_METADATA_FORMAT, @@ -530,11 +421,20 @@ private UploadedMetadataResults writeMetadataInParallel( String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key); uploadTasks.put( customComponent, - getAsyncMetadataWriteAction(clusterState, customComponent, CUSTOM_METADATA_FORMAT, value, listener) + remoteGlobalMetadataManager.getAsyncMetadataWriteAction( + clusterState, + customComponent, + CUSTOM_METADATA_FORMAT, + value, + listener + ) ); }); indexToUpload.forEach(indexMetadata -> { - uploadTasks.put(indexMetadata.getIndexName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); + uploadTasks.put( + indexMetadata.getIndexName(), + remoteIndexMetadataManager.getIndexMetadataAsyncAction(clusterState, indexMetadata, listener) + ); }); // start async upload of all required metadata files @@ -543,7 +443,7 @@ private UploadedMetadataResults writeMetadataInParallel( } try { - if (latch.await(getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + if (latch.await(remoteGlobalMetadataManager.getGlobalMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { // TODO: We should add metrics where transfer is timing out. [Issue: #10687] RemoteStateTransferException ex = new RemoteStateTransferException( String.format( @@ -603,75 +503,6 @@ private UploadedMetadataResults writeMetadataInParallel( return response; } - /** - * Allows async Upload of IndexMetadata to remote - * - * @param clusterState current ClusterState - * @param indexMetadata {@link IndexMetadata} to upload - * @param latchedActionListener listener to respond back on after upload finishes - */ - private CheckedRunnable getIndexMetadataAsyncAction( - ClusterState clusterState, - IndexMetadata indexMetadata, - LatchedActionListener latchedActionListener - ) { - final BlobContainer indexMetadataContainer = indexMetadataContainer( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID(), - indexMetadata.getIndexUUID() - ); - final String indexMetadataFilename = indexMetadataFileName(indexMetadata); - ActionListener completionListener = ActionListener.wrap( - resp -> latchedActionListener.onResponse( - new UploadedIndexMetadata( - indexMetadata.getIndex().getName(), - indexMetadata.getIndexUUID(), - indexMetadataContainer.path().buildAsString() + indexMetadataFilename - ) - ), - ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) - ); - - return () -> INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( - indexMetadata, - indexMetadataContainer, - indexMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); - } - - /** - * Allows async upload of Metadata components to remote - */ - - private CheckedRunnable getAsyncMetadataWriteAction( - ClusterState clusterState, - String component, - ChecksumBlobStoreFormat componentMetadataBlobStore, - ToXContent componentMetadata, - LatchedActionListener latchedActionListener - ) { - final BlobContainer globalMetadataContainer = globalMetadataContainer( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() - ); - final String componentMetadataFilename = metadataAttributeFileName(component, clusterState.metadata().version()); - ActionListener completionListener = ActionListener.wrap( - resp -> latchedActionListener.onResponse(new UploadedMetadataAttribute(component, componentMetadataFilename)), - ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, ex)) - ); - return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority( - componentMetadata, - globalMetadataContainer, - componentMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); - } - @Nullable public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) throws IOException { @@ -681,7 +512,7 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat return null; } assert previousManifest != null : "Last cluster metadata manifest is not set"; - ClusterMetadataManifest committedManifest = uploadManifest( + ClusterMetadataManifest committedManifest = remoteManifestManager.uploadManifest( clusterState, previousManifest.getIndices(), previousManifest.getPreviousClusterUUID(), @@ -695,6 +526,17 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat return committedManifest; } + /** + * Fetch latest ClusterMetadataManifest from remote state store + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @return ClusterMetadataManifest + */ + public Optional getLatestClusterMetadataManifest(String clusterName, String clusterUUID) { + return remoteManifestManager.getLatestClusterMetadataManifest(clusterName, clusterUUID); + } + @Override public void close() throws IOException { if (blobStoreRepository != null) { @@ -711,131 +553,16 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; - } - - private ClusterMetadataManifest uploadV1Manifest( - ClusterState clusterState, - List uploadedIndexMetadata, - String previousClusterUUID, - String globalMetadataFileName, - boolean committed - ) throws IOException { - synchronized (this) { - final String manifestFileName = getManifestFileName( - clusterState.term(), - clusterState.version(), - committed, - ClusterMetadataManifest.CODEC_V1 - ); - ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() - .clusterTerm(clusterState.term()) - .stateVersion(clusterState.getVersion()) - .clusterUUID(clusterState.metadata().clusterUUID()) - .stateUUID(clusterState.stateUUID()) - .opensearchVersion(Version.CURRENT) - .nodeId(nodeId) - .committed(committed) - .codecVersion(ClusterMetadataManifest.CODEC_V1) - .globalMetadataFileName(globalMetadataFileName) - .indices(uploadedIndexMetadata) - .previousClusterUUID(previousClusterUUID) - .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) - .build(); - writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); - return manifest; - } - } - - private ClusterMetadataManifest uploadManifest( - ClusterState clusterState, - List uploadedIndexMetadata, - String previousClusterUUID, - UploadedMetadataAttribute uploadedCoordinationMetadata, - UploadedMetadataAttribute uploadedSettingsMetadata, - UploadedMetadataAttribute uploadedTemplatesMetadata, - Map uploadedCustomMetadataMap, - boolean committed - ) throws IOException { - synchronized (this) { - final String manifestFileName = getManifestFileName( - clusterState.term(), - clusterState.version(), - committed, - MANIFEST_CURRENT_CODEC_VERSION - ); - final ClusterMetadataManifest manifest = new ClusterMetadataManifest( - clusterState.term(), - clusterState.getVersion(), - clusterState.metadata().clusterUUID(), - clusterState.stateUUID(), - Version.CURRENT, - nodeId, - committed, - MANIFEST_CURRENT_CODEC_VERSION, - null, - uploadedIndexMetadata, - previousClusterUUID, - clusterState.metadata().clusterUUIDCommitted(), - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplatesMetadata, - uploadedCustomMetadataMap - ); - writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); - return manifest; - } - } - - private void writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest, String fileName) - throws IOException { - AtomicReference result = new AtomicReference(); - AtomicReference exceptionReference = new AtomicReference(); - - final BlobContainer metadataManifestContainer = manifestContainer(clusterName, clusterUUID); - - // latch to wait until upload is not finished - CountDownLatch latch = new CountDownLatch(1); - - LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> { - logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); - }, ex -> { exceptionReference.set(ex); }), latch); - - getClusterMetadataManifestBlobStoreFormat(fileName).writeAsyncWithUrgentPriority( - uploadManifest, - metadataManifestContainer, - fileName, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); - - try { - if (latch.await(getMetadataManifestUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { - RemoteStateTransferException ex = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of manifest file to complete") - ); - throw ex; - } - } catch (InterruptedException ex) { - RemoteStateTransferException exception = new RemoteStateTransferException( - String.format(Locale.ROOT, "Timed out waiting for transfer of manifest file to complete - %s"), - ex - ); - Thread.currentThread().interrupt(); - throw exception; - } - if (exceptionReference.get() != null) { - throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); - } - logger.debug( - "Metadata manifest file [{}] written during [{}] phase. ", - fileName, - uploadManifest.isCommitted() ? "commit" : "publish" - ); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings); + remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); } private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) { - final Optional latestManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID); + final Optional latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( + clusterName, + clusterUUID + ); if (!latestManifest.isPresent()) { final String previousClusterUUID = getLastKnownUUIDFromRemote(clusterName); assert !clusterUUID.equals(previousClusterUUID) : "Last cluster UUID is same current cluster UUID"; @@ -844,190 +571,10 @@ private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) return latestManifest.get().getPreviousClusterUUID(); } - private BlobContainer indexMetadataContainer(String clusterName, String clusterUUID, String indexUUID) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX - return blobStoreRepository.blobStore() - .blobContainer(getCusterMetadataBasePath(clusterName, clusterUUID).add(INDEX_PATH_TOKEN).add(indexUUID)); - } - - private BlobContainer globalMetadataContainer(String clusterName, String clusterUUID) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/ - return blobStoreRepository.blobStore() - .blobContainer(getCusterMetadataBasePath(clusterName, clusterUUID).add(GLOBAL_METADATA_PATH_TOKEN)); - } - - private BlobContainer manifestContainer(String clusterName, String clusterUUID) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest - return blobStoreRepository.blobStore().blobContainer(getManifestFolderPath(clusterName, clusterUUID)); - } - - private BlobPath getCusterMetadataBasePath(String clusterName, String clusterUUID) { - return blobStoreRepository.basePath().add(encodeString(clusterName)).add(CLUSTER_STATE_PATH_TOKEN).add(clusterUUID); - } - - private BlobContainer clusterUUIDContainer(String clusterName) { - return blobStoreRepository.blobStore() - .blobContainer( - blobStoreRepository.basePath() - .add(Base64.getUrlEncoder().withoutPadding().encodeToString(clusterName.getBytes(StandardCharsets.UTF_8))) - .add(CLUSTER_STATE_PATH_TOKEN) - ); - } - private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { this.slowWriteLoggingThreshold = slowWriteLoggingThreshold; } - private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) { - this.indexMetadataUploadTimeout = newIndexMetadataUploadTimeout; - } - - private void setGlobalMetadataUploadTimeout(TimeValue newGlobalMetadataUploadTimeout) { - this.globalMetadataUploadTimeout = newGlobalMetadataUploadTimeout; - } - - private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploadTimeout) { - this.metadataManifestUploadTimeout = newMetadataManifestUploadTimeout; - } - - private Map getUpdatedCustoms(ClusterState currentState, ClusterState previousState) { - if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) { - return new HashMap<>(); - } - Map updatedCustom = new HashMap<>(); - Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); - for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { - if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { - if (currentCustoms.contains(cursor.getKey()) - && !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { - // If the custom metadata is updated, we need to upload the new version. - updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); - } - currentCustoms.remove(cursor.getKey()); - } - } - for (String custom : currentCustoms) { - Metadata.Custom cursor = currentState.metadata().custom(custom); - if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) { - updatedCustom.put(custom, cursor); - } - } - return updatedCustom; - } - - public TimeValue getIndexMetadataUploadTimeout() { - return this.indexMetadataUploadTimeout; - } - - public TimeValue getGlobalMetadataUploadTimeout() { - return this.globalMetadataUploadTimeout; - } - - public TimeValue getMetadataManifestUploadTimeout() { - return this.metadataManifestUploadTimeout; - } - - static String getManifestFileName(long term, long version, boolean committed, int codecVersion) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ - return String.join( - DELIMITER, - MANIFEST_PATH_TOKEN, - RemoteStoreUtils.invertLong(term), - RemoteStoreUtils.invertLong(version), - (committed ? "C" : "P"), // C for committed and P for published - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(codecVersion) // Keep the codec version at last place only, during read we reads last place to - // determine codec version. - ); - } - - static String indexMetadataFileName(IndexMetadata indexMetadata) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index//metadata______ - return String.join( - DELIMITER, - METADATA_FILE_PREFIX, - RemoteStoreUtils.invertLong(indexMetadata.getVersion()), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last - // place to determine codec version. - ); - } - - private static String globalMetadataFileName(Metadata metadata) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/metadata______ - return String.join( - DELIMITER, - METADATA_FILE_PREFIX, - RemoteStoreUtils.invertLong(metadata.version()), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) - ); - } - - private static String metadataAttributeFileName(String componentPrefix, Long metadataVersion) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ - return String.join( - DELIMITER, - componentPrefix, - RemoteStoreUtils.invertLong(metadataVersion), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) - ); - } - - private BlobPath getManifestFolderPath(String clusterName, String clusterUUID) { - return getCusterMetadataBasePath(clusterName, clusterUUID).add(MANIFEST_PATH_TOKEN); - } - - /** - * Fetch latest index metadata from remote cluster state - * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster - * @param clusterMetadataManifest manifest file of cluster - * @return {@code Map} latest IndexUUID to IndexMetadata map - */ - private Map getIndexMetadataMap( - String clusterName, - String clusterUUID, - ClusterMetadataManifest clusterMetadataManifest - ) { - assert Objects.equals(clusterUUID, clusterMetadataManifest.getClusterUUID()) - : "Corrupt ClusterMetadataManifest found. Cluster UUID mismatch."; - Map remoteIndexMetadata = new HashMap<>(); - for (UploadedIndexMetadata uploadedIndexMetadata : clusterMetadataManifest.getIndices()) { - IndexMetadata indexMetadata = getIndexMetadata(clusterName, clusterUUID, uploadedIndexMetadata); - remoteIndexMetadata.put(uploadedIndexMetadata.getIndexUUID(), indexMetadata); - } - return remoteIndexMetadata; - } - - /** - * Fetch index metadata from remote cluster state - * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster - * @param uploadedIndexMetadata {@link UploadedIndexMetadata} contains details about remote location of index metadata - * @return {@link IndexMetadata} - */ - private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, UploadedIndexMetadata uploadedIndexMetadata) { - BlobContainer blobContainer = indexMetadataContainer(clusterName, clusterUUID, uploadedIndexMetadata.getIndexUUID()); - try { - String[] splitPath = uploadedIndexMetadata.getUploadedFilename().split("/"); - return INDEX_METADATA_FORMAT.read( - blobContainer, - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading IndexMetadata - %s", uploadedIndexMetadata.getUploadedFilename()), - e - ); - } - } - /** * Fetch latest ClusterState from remote, including global metadata, index metadata and cluster state version * @@ -1037,7 +584,10 @@ private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, U */ public ClusterState getLatestClusterState(String clusterName, String clusterUUID) { start(); - Optional clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID); + Optional clusterMetadataManifest = remoteManifestManager.getLatestClusterMetadataManifest( + clusterName, + clusterUUID + ); if (clusterMetadataManifest.isEmpty()) { throw new IllegalStateException( String.format(Locale.ROOT, "Latest cluster metadata manifest is not present for the provided clusterUUID: %s", clusterUUID) @@ -1045,10 +595,14 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID } // Fetch Global Metadata - Metadata globalMetadata = getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get()); + Metadata globalMetadata = remoteGlobalMetadataManager.getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get()); // Fetch Index Metadata - Map indices = getIndexMetadataMap(clusterName, clusterUUID, clusterMetadataManifest.get()); + Map indices = remoteIndexMetadataManager.getIndexMetadataMap( + clusterName, + clusterUUID, + clusterMetadataManifest.get() + ); Map indexMetadataMap = new HashMap<>(); indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); }); @@ -1059,154 +613,6 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID .build(); } - private Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { - String globalMetadataFileName = clusterMetadataManifest.getGlobalMetadataFileName(); - try { - // Fetch Global metadata - if (globalMetadataFileName != null) { - String[] splitPath = globalMetadataFileName.split("/"); - return GLOBAL_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); - } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { - CoordinationMetadata coordinationMetadata = getCoordinationMetadata( - clusterName, - clusterUUID, - clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() - ); - Settings settingsMetadata = getSettingsMetadata( - clusterName, - clusterUUID, - clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() - ); - TemplatesMetadata templatesMetadata = getTemplatesMetadata( - clusterName, - clusterUUID, - clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() - ); - Metadata.Builder builder = new Metadata.Builder(); - builder.coordinationMetadata(coordinationMetadata); - builder.persistentSettings(settingsMetadata); - builder.templates(templatesMetadata); - clusterMetadataManifest.getCustomMetadataMap() - .forEach( - (key, value) -> builder.putCustom( - key, - getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key) - ) - ); - return builder.build(); - } else { - return Metadata.EMPTY_METADATA; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Global Metadata - %s", globalMetadataFileName), - e - ); - } - } - - private CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { - try { - // Fetch Coordination metadata - if (coordinationMetadataFileName != null) { - String[] splitPath = coordinationMetadataFileName.split("/"); - return COORDINATION_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); - } else { - return CoordinationMetadata.EMPTY_METADATA; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Coordination Metadata - %s", coordinationMetadataFileName), - e - ); - } - } - - private Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { - try { - // Fetch Settings metadata - if (settingsMetadataFileName != null) { - String[] splitPath = settingsMetadataFileName.split("/"); - return SETTINGS_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); - } else { - return Settings.EMPTY; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Settings Metadata - %s", settingsMetadataFileName), - e - ); - } - } - - private TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { - try { - // Fetch Templates metadata - if (templatesMetadataFileName != null) { - String[] splitPath = templatesMetadataFileName.split("/"); - return TEMPLATES_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); - } else { - return TemplatesMetadata.EMPTY_METADATA; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", templatesMetadataFileName), - e - ); - } - } - - private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { - requireNonNull(customMetadataFileName); - try { - // Fetch Custom metadata - String[] splitPath = customMetadataFileName.split("/"); - ChecksumBlobStoreFormat customChecksumBlobStoreFormat = new ChecksumBlobStoreFormat<>( - "custom", - METADATA_NAME_FORMAT, - (parser -> Metadata.Custom.fromXContent(parser, custom)) - ); - return customChecksumBlobStoreFormat.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), - e - ); - } - } - - /** - * Fetch latest ClusterMetadataManifest from remote state store - * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster - * @return ClusterMetadataManifest - */ - public Optional getLatestClusterMetadataManifest(String clusterName, String clusterUUID) { - Optional latestManifestFileName = getLatestManifestFileName(clusterName, clusterUUID); - return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); - } - /** * Fetch the previous cluster UUIDs from remote state store and return the most recent valid cluster UUID * @@ -1216,7 +622,10 @@ public Optional getLatestClusterMetadataManifest(String public String getLastKnownUUIDFromRemote(String clusterName) { try { Set clusterUUIDs = getAllClusterUUIDs(clusterName); - Map latestManifests = getLatestManifestForAllClusterUUIDs(clusterName, clusterUUIDs); + Map latestManifests = remoteManifestManager.getLatestManifestForAllClusterUUIDs( + clusterName, + clusterUUIDs + ); List validChain = createClusterChain(latestManifests, clusterName); if (validChain.isEmpty()) { return ClusterState.UNKNOWN_UUID; @@ -1231,29 +640,13 @@ public String getLastKnownUUIDFromRemote(String clusterName) { } private Set getAllClusterUUIDs(String clusterName) throws IOException { - Map clusterUUIDMetadata = clusterUUIDContainer(clusterName).children(); + Map clusterUUIDMetadata = clusterUUIDContainer(blobStoreRepository, clusterName).children(); if (clusterUUIDMetadata == null) { return Collections.emptySet(); } return Collections.unmodifiableSet(clusterUUIDMetadata.keySet()); } - private Map getLatestManifestForAllClusterUUIDs(String clusterName, Set clusterUUIDs) { - Map manifestsByClusterUUID = new HashMap<>(); - for (String clusterUUID : clusterUUIDs) { - try { - Optional manifest = getLatestClusterMetadataManifest(clusterName, clusterUUID); - manifest.ifPresent(clusterMetadataManifest -> manifestsByClusterUUID.put(clusterUUID, clusterMetadataManifest)); - } catch (Exception e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Exception in fetching manifest for clusterUUID: %s", clusterUUID), - e - ); - } - } - return manifestsByClusterUUID; - } - /** * This method creates a valid cluster UUID chain. * @@ -1331,7 +724,7 @@ private Map trimClusterUUIDs( if (!ClusterState.UNKNOWN_UUID.equals(currentManifest.getPreviousClusterUUID())) { ClusterMetadataManifest previousManifest = trimmedUUIDs.get(currentManifest.getPreviousClusterUUID()); if (isMetadataEqual(currentManifest, previousManifest, clusterName) - && isGlobalMetadataEqual(currentManifest, previousManifest, clusterName)) { + && remoteGlobalMetadataManager.isGlobalMetadataEqual(currentManifest, previousManifest, clusterName)) { trimmedUUIDs.remove(clusterUUID); } } @@ -1348,12 +741,20 @@ private boolean isMetadataEqual(ClusterMetadataManifest first, ClusterMetadataMa .stream() .collect(Collectors.toMap(md -> md.getIndexName(), Function.identity())); for (UploadedIndexMetadata uploadedIndexMetadata : first.getIndices()) { - final IndexMetadata firstIndexMetadata = getIndexMetadata(clusterName, first.getClusterUUID(), uploadedIndexMetadata); + final IndexMetadata firstIndexMetadata = remoteIndexMetadataManager.getIndexMetadata( + clusterName, + first.getClusterUUID(), + uploadedIndexMetadata + ); final UploadedIndexMetadata secondUploadedIndexMetadata = secondIndices.get(uploadedIndexMetadata.getIndexName()); if (secondUploadedIndexMetadata == null) { return false; } - final IndexMetadata secondIndexMetadata = getIndexMetadata(clusterName, second.getClusterUUID(), secondUploadedIndexMetadata); + final IndexMetadata secondIndexMetadata = remoteIndexMetadataManager.getIndexMetadata( + clusterName, + second.getClusterUUID(), + secondUploadedIndexMetadata + ); if (firstIndexMetadata.equals(secondIndexMetadata) == false) { return false; } @@ -1361,125 +762,14 @@ private boolean isMetadataEqual(ClusterMetadataManifest first, ClusterMetadataMa return true; } - private boolean isGlobalMetadataEqual(ClusterMetadataManifest first, ClusterMetadataManifest second, String clusterName) { - Metadata secondGlobalMetadata = getGlobalMetadata(clusterName, second.getClusterUUID(), second); - Metadata firstGlobalMetadata = getGlobalMetadata(clusterName, first.getClusterUUID(), first); - return Metadata.isGlobalResourcesMetadataEquals(firstGlobalMetadata, secondGlobalMetadata); - } - private boolean isValidClusterUUID(ClusterMetadataManifest manifest) { return manifest.isClusterUUIDCommitted(); } - /** - * Fetch ClusterMetadataManifest files from remote state store in order - * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster - * @param limit max no of files to fetch - * @return all manifest file names - */ - private List getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException { - try { - - /* - {@link BlobContainer#listBlobsByPrefixInSortedOrder} will list the latest manifest file first - as the manifest file name generated via {@link RemoteClusterStateService#getManifestFileName} ensures - when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top. - */ - return manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder( - MANIFEST_FILE_PREFIX + DELIMITER, - limit, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ); - } catch (IOException e) { - throw new IllegalStateException("Error while fetching latest manifest file for remote cluster state", e); - } - } - - /** - * Fetch latest ClusterMetadataManifest file from remote state store - * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster - * @return latest ClusterMetadataManifest filename - */ - private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { - List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, 1); - if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { - return Optional.of(manifestFilesMetadata.get(0).name()); - } - logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); - return Optional.empty(); - } - - /** - * Fetch ClusterMetadataManifest from remote state store - * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster - * @return ClusterMetadataManifest - */ - private ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, String clusterUUID, String filename) - throws IllegalStateException { - try { - return getClusterMetadataManifestBlobStoreFormat(filename).read( - manifestContainer(clusterName, clusterUUID), - filename, - blobStoreRepository.getNamedXContentRegistry() - ); - } catch (IOException e) { - throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); - } - } - - private ChecksumBlobStoreFormat getClusterMetadataManifestBlobStoreFormat(String fileName) { - long codecVersion = getManifestCodecVersion(fileName); - if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { - return CLUSTER_METADATA_MANIFEST_FORMAT; - } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { - return CLUSTER_METADATA_MANIFEST_FORMAT_V1; - } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { - return CLUSTER_METADATA_MANIFEST_FORMAT_V0; - } - - throw new IllegalArgumentException("Cluster metadata manifest file is corrupted, don't have valid codec version"); - } - - private int getManifestCodecVersion(String fileName) { - String[] splitName = fileName.split(DELIMITER); - if (splitName.length == SPLITED_MANIFEST_FILE_LENGTH) { - return Integer.parseInt(splitName[splitName.length - 1]); // Last value would be codec version. - } else if (splitName.length < SPLITED_MANIFEST_FILE_LENGTH) { // Where codec is not part of file name, i.e. default codec version 0 - // is used. - return ClusterMetadataManifest.CODEC_V0; - } else { - throw new IllegalArgumentException("Manifest file name is corrupted"); - } - } - - public static String encodeString(String content) { - return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8)); - } - public void writeMetadataFailed() { getStats().stateFailed(); } - /** - * Exception for Remote state transfer. - */ - static class RemoteStateTransferException extends RuntimeException { - - public RemoteStateTransferException(String errorDesc) { - super(errorDesc); - } - - public RemoteStateTransferException(String errorDesc, Throwable cause) { - super(errorDesc, cause); - } - } - /** * Purges all remote cluster state against provided cluster UUIDs * @@ -1490,7 +780,7 @@ void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUI clusterUUIDs.forEach(clusterUUID -> { getBlobStoreTransferService().deleteAsync( ThreadPool.Names.REMOTE_PURGE, - getCusterMetadataBasePath(clusterName, clusterUUID), + getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID), new ActionListener<>() { @Override public void onResponse(Void unused) { @@ -1529,7 +819,7 @@ void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int mani try { getBlobStoreTransferService().listAllInSortedOrderAsync( ThreadPool.Names.REMOTE_PURGE, - getManifestFolderPath(clusterName, clusterUUID), + remoteManifestManager.getManifestFolderPath(clusterName, clusterUUID), "manifest", Integer.MAX_VALUE, new ActionListener<>() { @@ -1576,7 +866,7 @@ private void deleteClusterMetadata( Set staleIndexMetadataPaths = new HashSet<>(); Set staleGlobalMetadataPaths = new HashSet<>(); activeManifestBlobMetadata.forEach(blobMetadata -> { - ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest( + ClusterMetadataManifest clusterMetadataManifest = remoteManifestManager.fetchRemoteClusterMetadataManifest( clusterName, clusterUUID, blobMetadata.name() @@ -1594,7 +884,7 @@ private void deleteClusterMetadata( } }); staleManifestBlobMetadata.forEach(blobMetadata -> { - ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest( + ClusterMetadataManifest clusterMetadataManifest = remoteManifestManager.fetchRemoteClusterMetadataManifest( clusterName, clusterUUID, blobMetadata.name() @@ -1681,7 +971,7 @@ private void deleteClusterMetadata( private void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { logger.debug(String.format(Locale.ROOT, "Deleting stale files from remote - %s", stalePaths)); - getBlobStoreTransferService().deleteBlobs(getCusterMetadataBasePath(clusterName, clusterUUID), stalePaths); + getBlobStoreTransferService().deleteBlobs(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID), stalePaths); } /** @@ -1710,34 +1000,4 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa public RemotePersistenceStats getStats() { return remoteStateStats; } - - private static class UploadedMetadataResults { - List uploadedIndexMetadata; - Map uploadedCustomMetadataMap; - UploadedMetadataAttribute uploadedCoordinationMetadata; - UploadedMetadataAttribute uploadedSettingsMetadata; - UploadedMetadataAttribute uploadedTemplatesMetadata; - - public UploadedMetadataResults( - List uploadedIndexMetadata, - Map uploadedCustomMetadataMap, - UploadedMetadataAttribute uploadedCoordinationMetadata, - UploadedMetadataAttribute uploadedSettingsMetadata, - UploadedMetadataAttribute uploadedTemplatesMetadata - ) { - this.uploadedIndexMetadata = uploadedIndexMetadata; - this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; - this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; - this.uploadedSettingsMetadata = uploadedSettingsMetadata; - this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; - } - - public UploadedMetadataResults() { - this.uploadedIndexMetadata = new ArrayList<>(); - this.uploadedCustomMetadataMap = new HashMap<>(); - this.uploadedCoordinationMetadata = null; - this.uploadedSettingsMetadata = null; - this.uploadedTemplatesMetadata = null; - } - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java new file mode 100644 index 0000000000000..4e8fb7cf351ea --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.repositories.blobstore.BlobStoreRepository; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RemoteClusterStateUtils { + public static final String METADATA_NAME_FORMAT = "%s.dat"; + public static final String METADATA_FILE_PREFIX = "metadata"; + public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; + public static final String DELIMITER = "__"; + + // ToXContent Params with gateway mode. + // We are using gateway context mode to persist all custom metadata. + public static final ToXContent.Params FORMAT_PARAMS; + static { + Map params = new HashMap<>(1); + params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY); + FORMAT_PARAMS = new ToXContent.MapParams(params); + } + + public static BlobPath getCusterMetadataBasePath(BlobStoreRepository blobStoreRepository, String clusterName, String clusterUUID) { + return blobStoreRepository.basePath().add(encodeString(clusterName)).add(CLUSTER_STATE_PATH_TOKEN).add(clusterUUID); + } + + public static String encodeString(String content) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8)); + } + + static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepository, String clusterName) { + return blobStoreRepository.blobStore() + .blobContainer( + blobStoreRepository.basePath() + .add(Base64.getUrlEncoder().withoutPadding().encodeToString(clusterName.getBytes(StandardCharsets.UTF_8))) + .add(CLUSTER_STATE_PATH_TOKEN) + ); + } + + /** + * Exception for Remote state transfer. + */ + static class RemoteStateTransferException extends RuntimeException { + + public RemoteStateTransferException(String errorDesc) { + super(errorDesc); + } + + public RemoteStateTransferException(String errorDesc, Throwable cause) { + super(errorDesc, cause); + } + } + + static class UploadedMetadataResults { + List uploadedIndexMetadata; + Map uploadedCustomMetadataMap; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata; + + public UploadedMetadataResults( + List uploadedIndexMetadata, + Map uploadedCustomMetadataMap, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata + ) { + this.uploadedIndexMetadata = uploadedIndexMetadata; + this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; + this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; + this.uploadedSettingsMetadata = uploadedSettingsMetadata; + this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + } + + public UploadedMetadataResults() { + this.uploadedIndexMetadata = new ArrayList<>(); + this.uploadedCustomMetadataMap = new HashMap<>(); + this.uploadedCoordinationMetadata = null; + this.uploadedSettingsMetadata = null; + this.uploadedTemplatesMetadata = null; + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java new file mode 100644 index 0000000000000..9141085b6d5a2 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -0,0 +1,334 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static java.util.Objects.requireNonNull; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; + +public class RemoteGlobalMetadataManager { + public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; + public static final String COORDINATION_METADATA = "coordination"; + public static final String SETTING_METADATA = "settings"; + public static final String TEMPLATES_METADATA = "templates"; + public static final String CUSTOM_METADATA = "custom"; + public static final String CUSTOM_DELIMITER = "--"; + + public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); + + public static final Setting GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( + "cluster.remote_store.state.global_metadata.upload_timeout", + GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public static final ChecksumBlobStoreFormat GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "metadata", + METADATA_NAME_FORMAT, + Metadata::fromXContent + ); + + public static final ChecksumBlobStoreFormat COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "coordination", + METADATA_NAME_FORMAT, + CoordinationMetadata::fromXContent + ); + + public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "settings", + METADATA_NAME_FORMAT, + Settings::fromXContent + ); + + public static final ChecksumBlobStoreFormat TEMPLATES_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "templates", + METADATA_NAME_FORMAT, + TemplatesMetadata::fromXContent + ); + + public static final ChecksumBlobStoreFormat CUSTOM_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "custom", + METADATA_NAME_FORMAT, + Metadata.Custom::fromXContent + ); + public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; + + private final BlobStoreRepository blobStoreRepository; + + private volatile TimeValue globalMetadataUploadTimeout; + + RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings) { + this.blobStoreRepository = blobStoreRepository; + this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); + clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); + } + + /** + * Allows async upload of Metadata components to remote + */ + CheckedRunnable getAsyncMetadataWriteAction( + ClusterState clusterState, + String component, + ChecksumBlobStoreFormat componentMetadataBlobStore, + ToXContent componentMetadata, + LatchedActionListener latchedActionListener + ) { + final BlobContainer globalMetadataContainer = globalMetadataContainer( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID() + ); + final String componentMetadataFilename = metadataAttributeFileName(component, clusterState.metadata().version()); + ActionListener completionListener = ActionListener.wrap( + resp -> latchedActionListener.onResponse( + new ClusterMetadataManifest.UploadedMetadataAttribute(component, componentMetadataFilename) + ), + ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, ex)) + ); + return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority( + componentMetadata, + globalMetadataContainer, + componentMetadataFilename, + blobStoreRepository.getCompressor(), + completionListener, + FORMAT_PARAMS + ); + } + + Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { + String globalMetadataFileName = clusterMetadataManifest.getGlobalMetadataFileName(); + try { + // Fetch Global metadata + if (globalMetadataFileName != null) { + String[] splitPath = globalMetadataFileName.split("/"); + return GLOBAL_METADATA_FORMAT.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata( + clusterName, + clusterUUID, + clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() + ); + Settings settingsMetadata = getSettingsMetadata( + clusterName, + clusterUUID, + clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() + ); + TemplatesMetadata templatesMetadata = getTemplatesMetadata( + clusterName, + clusterUUID, + clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() + ); + Metadata.Builder builder = new Metadata.Builder(); + builder.coordinationMetadata(coordinationMetadata); + builder.persistentSettings(settingsMetadata); + builder.templates(templatesMetadata); + clusterMetadataManifest.getCustomMetadataMap() + .forEach( + (key, value) -> builder.putCustom( + key, + getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key) + ) + ); + return builder.build(); + } else { + return Metadata.EMPTY_METADATA; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Global Metadata - %s", globalMetadataFileName), + e + ); + } + } + + private CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { + try { + // Fetch Coordination metadata + if (coordinationMetadataFileName != null) { + String[] splitPath = coordinationMetadataFileName.split("/"); + return COORDINATION_METADATA_FORMAT.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else { + return CoordinationMetadata.EMPTY_METADATA; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Coordination Metadata - %s", coordinationMetadataFileName), + e + ); + } + } + + private Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { + try { + // Fetch Settings metadata + if (settingsMetadataFileName != null) { + String[] splitPath = settingsMetadataFileName.split("/"); + return SETTINGS_METADATA_FORMAT.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else { + return Settings.EMPTY; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Settings Metadata - %s", settingsMetadataFileName), + e + ); + } + } + + private TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { + try { + // Fetch Templates metadata + if (templatesMetadataFileName != null) { + String[] splitPath = templatesMetadataFileName.split("/"); + return TEMPLATES_METADATA_FORMAT.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else { + return TemplatesMetadata.EMPTY_METADATA; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", templatesMetadataFileName), + e + ); + } + } + + private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { + requireNonNull(customMetadataFileName); + try { + // Fetch Custom metadata + String[] splitPath = customMetadataFileName.split("/"); + ChecksumBlobStoreFormat customChecksumBlobStoreFormat = new ChecksumBlobStoreFormat<>( + "custom", + METADATA_NAME_FORMAT, + (parser -> Metadata.Custom.fromXContent(parser, custom)) + ); + return customChecksumBlobStoreFormat.read( + globalMetadataContainer(clusterName, clusterUUID), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), + e + ); + } + } + + Map getUpdatedCustoms(ClusterState currentState, ClusterState previousState) { + if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) { + return new HashMap<>(); + } + Map updatedCustom = new HashMap<>(); + Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); + for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { + if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { + if (currentCustoms.contains(cursor.getKey()) + && !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { + // If the custom metadata is updated, we need to upload the new version. + updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); + } + currentCustoms.remove(cursor.getKey()); + } + } + for (String custom : currentCustoms) { + Metadata.Custom cursor = currentState.metadata().custom(custom); + if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) { + updatedCustom.put(custom, cursor); + } + } + return updatedCustom; + } + + private BlobContainer globalMetadataContainer(String clusterName, String clusterUUID) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/ + return blobStoreRepository.blobStore() + .blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(GLOBAL_METADATA_PATH_TOKEN)); + } + + private static String globalMetadataFileName(Metadata metadata) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/metadata______ + return String.join( + DELIMITER, + METADATA_FILE_PREFIX, + RemoteStoreUtils.invertLong(metadata.version()), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + } + + private static String metadataAttributeFileName(String componentPrefix, Long metadataVersion) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ + return String.join( + DELIMITER, + componentPrefix, + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + } + + boolean isGlobalMetadataEqual(ClusterMetadataManifest first, ClusterMetadataManifest second, String clusterName) { + Metadata secondGlobalMetadata = getGlobalMetadata(clusterName, second.getClusterUUID(), second); + Metadata firstGlobalMetadata = getGlobalMetadata(clusterName, first.getClusterUUID(), first); + return Metadata.isGlobalResourcesMetadataEquals(firstGlobalMetadata, secondGlobalMetadata); + } + + private void setGlobalMetadataUploadTimeout(TimeValue newGlobalMetadataUploadTimeout) { + this.globalMetadataUploadTimeout = newGlobalMetadataUploadTimeout; + } + + public TimeValue getGlobalMetadataUploadTimeout() { + return this.globalMetadataUploadTimeout; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java new file mode 100644 index 0000000000000..c034d597817ea --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; + +public class RemoteIndexMetadataManager { + public static final TimeValue INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); + + public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( + "cluster.remote_store.state.index_metadata.upload_timeout", + INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "index-metadata", + METADATA_NAME_FORMAT, + IndexMetadata::fromXContent + ); + public static final String INDEX_PATH_TOKEN = "index"; + + private final BlobStoreRepository blobStoreRepository; + + private volatile TimeValue indexMetadataUploadTimeout; + + public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings) { + this.blobStoreRepository = blobStoreRepository; + this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); + clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); + } + + /** + * Allows async Upload of IndexMetadata to remote + * + * @param clusterState current ClusterState + * @param indexMetadata {@link IndexMetadata} to upload + * @param latchedActionListener listener to respond back on after upload finishes + */ + CheckedRunnable getIndexMetadataAsyncAction( + ClusterState clusterState, + IndexMetadata indexMetadata, + LatchedActionListener latchedActionListener + ) { + final BlobContainer indexMetadataContainer = indexMetadataContainer( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID(), + indexMetadata.getIndexUUID() + ); + final String indexMetadataFilename = indexMetadataFileName(indexMetadata); + ActionListener completionListener = ActionListener.wrap( + resp -> latchedActionListener.onResponse( + new ClusterMetadataManifest.UploadedIndexMetadata( + indexMetadata.getIndex().getName(), + indexMetadata.getIndexUUID(), + indexMetadataContainer.path().buildAsString() + indexMetadataFilename + ) + ), + ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) + ); + + return () -> INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( + indexMetadata, + indexMetadataContainer, + indexMetadataFilename, + blobStoreRepository.getCompressor(), + completionListener, + FORMAT_PARAMS + ); + } + + /** + * Fetch index metadata from remote cluster state + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @param uploadedIndexMetadata {@link ClusterMetadataManifest.UploadedIndexMetadata} contains details about remote location of index metadata + * @return {@link IndexMetadata} + */ + IndexMetadata getIndexMetadata( + String clusterName, + String clusterUUID, + ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata + ) { + BlobContainer blobContainer = indexMetadataContainer(clusterName, clusterUUID, uploadedIndexMetadata.getIndexUUID()); + try { + String[] splitPath = uploadedIndexMetadata.getUploadedFilename().split("/"); + return INDEX_METADATA_FORMAT.read( + blobContainer, + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading IndexMetadata - %s", uploadedIndexMetadata.getUploadedFilename()), + e + ); + } + } + + /** + * Fetch latest index metadata from remote cluster state + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @param clusterMetadataManifest manifest file of cluster + * @return {@code Map} latest IndexUUID to IndexMetadata map + */ + Map getIndexMetadataMap( + String clusterName, + String clusterUUID, + ClusterMetadataManifest clusterMetadataManifest + ) { + assert Objects.equals(clusterUUID, clusterMetadataManifest.getClusterUUID()) + : "Corrupt ClusterMetadataManifest found. Cluster UUID mismatch."; + Map remoteIndexMetadata = new HashMap<>(); + for (ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata : clusterMetadataManifest.getIndices()) { + IndexMetadata indexMetadata = getIndexMetadata(clusterName, clusterUUID, uploadedIndexMetadata); + remoteIndexMetadata.put(uploadedIndexMetadata.getIndexUUID(), indexMetadata); + } + return remoteIndexMetadata; + } + + private BlobContainer indexMetadataContainer(String clusterName, String clusterUUID, String indexUUID) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX + return blobStoreRepository.blobStore() + .blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(INDEX_PATH_TOKEN).add(indexUUID)); + } + + public TimeValue getIndexMetadataUploadTimeout() { + return this.indexMetadataUploadTimeout; + } + + private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) { + this.indexMetadataUploadTimeout = newIndexMetadataUploadTimeout; + } + + static String indexMetadataFileName(IndexMetadata indexMetadata) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index//metadata______ + return String.join( + DELIMITER, + METADATA_FILE_PREFIX, + RemoteStoreUtils.invertLong(indexMetadata.getVersion()), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last + // place to determine codec version. + ); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java new file mode 100644 index 0000000000000..c741989552462 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -0,0 +1,357 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.Version; +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; + +public class RemoteManifestManager { + public static final String MANIFEST_PATH_TOKEN = "manifest"; + public static final String MANIFEST_FILE_PREFIX = "manifest"; + public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; + public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion + + public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); + + public static final Setting METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( + "cluster.remote_store.state.metadata_manifest.upload_timeout", + METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + /** + * Manifest format compatible with older codec v0, where codec version was missing. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V0 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); + /** + * Manifest format compatible with older codec v1, where global metadata was missing. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); + + /** + * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( + "cluster-metadata-manifest", + METADATA_MANIFEST_NAME_FORMAT, + ClusterMetadataManifest::fromXContent + ); + + private final BlobStoreRepository blobStoreRepository; + private volatile TimeValue metadataManifestUploadTimeout; + private final String nodeId; + private static final Logger logger = LogManager.getLogger(RemoteManifestManager.class); + + RemoteManifestManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, String nodeId) { + this.blobStoreRepository = blobStoreRepository; + this.metadataManifestUploadTimeout = clusterSettings.get(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING); + this.nodeId = nodeId; + clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); + } + + private ClusterMetadataManifest uploadV1Manifest( + ClusterState clusterState, + List uploadedIndexMetadata, + String previousClusterUUID, + String globalMetadataFileName, + boolean committed + ) throws IOException { + synchronized (this) { + final String manifestFileName = getManifestFileName( + clusterState.term(), + clusterState.version(), + committed, + ClusterMetadataManifest.CODEC_V1 + ); + ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(clusterState.term()) + .stateVersion(clusterState.getVersion()) + .clusterUUID(clusterState.metadata().clusterUUID()) + .stateUUID(clusterState.stateUUID()) + .opensearchVersion(Version.CURRENT) + .nodeId(nodeId) + .committed(committed) + .codecVersion(ClusterMetadataManifest.CODEC_V1) + .globalMetadataFileName(globalMetadataFileName) + .indices(uploadedIndexMetadata) + .previousClusterUUID(previousClusterUUID) + .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) + .build(); + writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); + return manifest; + } + } + + ClusterMetadataManifest uploadManifest( + ClusterState clusterState, + List uploadedIndexMetadata, + String previousClusterUUID, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, + Map uploadedCustomMetadataMap, + boolean committed + ) throws IOException { + synchronized (this) { + final String manifestFileName = getManifestFileName( + clusterState.term(), + clusterState.version(), + committed, + MANIFEST_CURRENT_CODEC_VERSION + ); + final ClusterMetadataManifest manifest = new ClusterMetadataManifest( + clusterState.term(), + clusterState.getVersion(), + clusterState.metadata().clusterUUID(), + clusterState.stateUUID(), + Version.CURRENT, + nodeId, + committed, + MANIFEST_CURRENT_CODEC_VERSION, + null, + uploadedIndexMetadata, + previousClusterUUID, + clusterState.metadata().clusterUUIDCommitted(), + uploadedCoordinationMetadata, + uploadedSettingsMetadata, + uploadedTemplatesMetadata, + uploadedCustomMetadataMap + ); + writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); + return manifest; + } + } + + private void writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest, String fileName) + throws IOException { + AtomicReference result = new AtomicReference(); + AtomicReference exceptionReference = new AtomicReference(); + + final BlobContainer metadataManifestContainer = manifestContainer(clusterName, clusterUUID); + + // latch to wait until upload is not finished + CountDownLatch latch = new CountDownLatch(1); + + LatchedActionListener completionListener = new LatchedActionListener<>(ActionListener.wrap(resp -> { + logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); + }, ex -> { exceptionReference.set(ex); }), latch); + + getClusterMetadataManifestBlobStoreFormat(fileName).writeAsyncWithUrgentPriority( + uploadManifest, + metadataManifestContainer, + fileName, + blobStoreRepository.getCompressor(), + completionListener, + FORMAT_PARAMS + ); + + try { + if (latch.await(getMetadataManifestUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { + RemoteStateTransferException ex = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of manifest file to complete") + ); + throw ex; + } + } catch (InterruptedException ex) { + RemoteStateTransferException exception = new RemoteStateTransferException( + String.format(Locale.ROOT, "Timed out waiting for transfer of manifest file to complete - %s"), + ex + ); + Thread.currentThread().interrupt(); + throw exception; + } + if (exceptionReference.get() != null) { + throw new RemoteStateTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); + } + logger.debug( + "Metadata manifest file [{}] written during [{}] phase. ", + fileName, + uploadManifest.isCommitted() ? "commit" : "publish" + ); + } + + /** + * Fetch latest ClusterMetadataManifest from remote state store + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @return ClusterMetadataManifest + */ + public Optional getLatestClusterMetadataManifest(String clusterName, String clusterUUID) { + Optional latestManifestFileName = getLatestManifestFileName(clusterName, clusterUUID); + return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); + } + + /** + * Fetch ClusterMetadataManifest from remote state store + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @return ClusterMetadataManifest + */ + ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, String clusterUUID, String filename) + throws IllegalStateException { + try { + return getClusterMetadataManifestBlobStoreFormat(filename).read( + manifestContainer(clusterName, clusterUUID), + filename, + blobStoreRepository.getNamedXContentRegistry() + ); + } catch (IOException e) { + throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); + } + } + + Map getLatestManifestForAllClusterUUIDs(String clusterName, Set clusterUUIDs) { + Map manifestsByClusterUUID = new HashMap<>(); + for (String clusterUUID : clusterUUIDs) { + try { + Optional manifest = getLatestClusterMetadataManifest(clusterName, clusterUUID); + manifest.ifPresent(clusterMetadataManifest -> manifestsByClusterUUID.put(clusterUUID, clusterMetadataManifest)); + } catch (Exception e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Exception in fetching manifest for clusterUUID: %s", clusterUUID), + e + ); + } + } + return manifestsByClusterUUID; + } + + private ChecksumBlobStoreFormat getClusterMetadataManifestBlobStoreFormat(String fileName) { + long codecVersion = getManifestCodecVersion(fileName); + if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { + return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V1; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V0; + } + + throw new IllegalArgumentException("Cluster metadata manifest file is corrupted, don't have valid codec version"); + } + + private int getManifestCodecVersion(String fileName) { + String[] splitName = fileName.split(DELIMITER); + if (splitName.length == SPLITED_MANIFEST_FILE_LENGTH) { + return Integer.parseInt(splitName[splitName.length - 1]); // Last value would be codec version. + } else if (splitName.length < SPLITED_MANIFEST_FILE_LENGTH) { // Where codec is not part of file name, i.e. default codec version 0 + // is used. + return ClusterMetadataManifest.CODEC_V0; + } else { + throw new IllegalArgumentException("Manifest file name is corrupted"); + } + } + + private BlobContainer manifestContainer(String clusterName, String clusterUUID) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest + return blobStoreRepository.blobStore().blobContainer(getManifestFolderPath(clusterName, clusterUUID)); + } + + BlobPath getManifestFolderPath(String clusterName, String clusterUUID) { + return getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(MANIFEST_PATH_TOKEN); + } + + public TimeValue getMetadataManifestUploadTimeout() { + return this.metadataManifestUploadTimeout; + } + + private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploadTimeout) { + this.metadataManifestUploadTimeout = newMetadataManifestUploadTimeout; + } + + /** + * Fetch ClusterMetadataManifest files from remote state store in order + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @param limit max no of files to fetch + * @return all manifest file names + */ + private List getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException { + try { + + /* + {@link BlobContainer#listBlobsByPrefixInSortedOrder} will list the latest manifest file first + as the manifest file name generated via {@link RemoteClusterStateService#getManifestFileName} ensures + when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top. + */ + return manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder( + MANIFEST_FILE_PREFIX + DELIMITER, + limit, + BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC + ); + } catch (IOException e) { + throw new IllegalStateException("Error while fetching latest manifest file for remote cluster state", e); + } + } + + static String getManifestFileName(long term, long version, boolean committed, int codecVersion) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ + return String.join( + DELIMITER, + MANIFEST_PATH_TOKEN, + RemoteStoreUtils.invertLong(term), + RemoteStoreUtils.invertLong(version), + (committed ? "C" : "P"), // C for committed and P for published + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(codecVersion) // Keep the codec version at last place only, during read we reads last place to + // determine codec version. + ); + } + + /** + * Fetch latest ClusterMetadataManifest file from remote state store + * + * @param clusterUUID uuid of cluster state to refer to in remote + * @param clusterName name of the cluster + * @return latest ClusterMetadataManifest filename + */ + private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, 1); + if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { + return Optional.of(manifestFilesMetadata.get(0).name()); + } + logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); + return Optional.empty(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 0b3cd49140939..94d71dc9eaa75 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -104,21 +104,22 @@ public void testClusterMetadataManifestXContent() throws IOException { Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", true, - new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), + new UploadedMetadataAttribute(RemoteGlobalMetadataManager.COORDINATION_METADATA, "coordination-file"), + new UploadedMetadataAttribute(RemoteGlobalMetadataManager.SETTING_METADATA, "setting-file"), + new UploadedMetadataAttribute(RemoteGlobalMetadataManager.TEMPLATES_METADATA, "templates-file"), Collections.unmodifiableList( Arrays.asList( new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, "custom--repositories-file" ), new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER + IndexGraveyard.TYPE, "custom--index_graveyard-file" ), new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER + WeightedRoutingMetadata.TYPE, "custom--weighted_routing_netadata-file" ) @@ -150,21 +151,22 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", true, - new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), + new UploadedMetadataAttribute(RemoteGlobalMetadataManager.COORDINATION_METADATA, "coordination-file"), + new UploadedMetadataAttribute(RemoteGlobalMetadataManager.SETTING_METADATA, "setting-file"), + new UploadedMetadataAttribute(RemoteGlobalMetadataManager.TEMPLATES_METADATA, "templates-file"), Collections.unmodifiableList( Arrays.asList( new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, + RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, "custom--repositories-file" ), new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER + IndexGraveyard.TYPE, "custom--index_graveyard-file" ), new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER + WeightedRoutingMetadata.TYPE, "custom--weighted_routing_netadata-file" ) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index e4b41d30ba677..0327ddc887f2e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -41,7 +41,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; -import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; @@ -85,16 +84,15 @@ import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; -import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; -import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; -import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -294,7 +292,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { .provideStream(0) .getInputStream() .readAllBytes(); - IndexMetadata writtenIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.deserialize( + IndexMetadata writtenIndexMetadata = RemoteIndexMetadataManager.INDEX_METADATA_FORMAT.deserialize( capturedWriteContext.get("metadata").getFileName(), blobStoreRepository.getNamedXContentRegistry(), new BytesArray(writtenBytes) @@ -333,7 +331,7 @@ public void run() { remoteClusterStateService.start(); assertThrows( - RemoteClusterStateService.RemoteStateTransferException.class, + RemoteStateTransferException.class, () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)) ); } @@ -366,7 +364,7 @@ public void testTimeoutWhileWritingManifestFile() throws IOException { try { remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)); } catch (Exception e) { - assertTrue(e instanceof RemoteClusterStateService.RemoteStateTransferException); + assertTrue(e instanceof RemoteStateTransferException); assertTrue(e.getMessage().contains("Timed out waiting for transfer of following metadata to complete")); } } @@ -387,7 +385,7 @@ public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOEx remoteClusterStateService.start(); assertThrows( - RemoteClusterStateService.RemoteStateTransferException.class, + RemoteStateTransferException.class, () -> remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)) ); assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); @@ -828,24 +826,6 @@ private void verifyMetadataAttributeOnlyUpdated( assertions.accept(initialManifest, manifestAfterMetadataUpdate); } - public void testReadLatestMetadataManifestFailedIOException() throws IOException { - final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); - - BlobContainer blobContainer = mockBlobStoreObjects(); - when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) - .thenThrow(IOException.class); - - remoteClusterStateService.start(); - Exception e = assertThrows( - IllegalStateException.class, - () -> remoteClusterStateService.getLatestClusterMetadataManifest( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() - ) - ); - assertEquals(e.getMessage(), "Error while fetching latest manifest file for remote cluster state"); - } - public void testReadLatestMetadataManifestFailedNoManifestFileInRemote() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); @@ -986,10 +966,10 @@ public void testReadGlobalMetadata() throws IOException { .stateUUID("state-uuid") .clusterUUID("cluster-uuid") .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) - .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) - .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) - .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) - .put(IndexGraveyard.TYPE, new UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) + .coordinationMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(COORDINATION_METADATA, "mock-coordination-file")) + .settingMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(SETTING_METADATA, "mock-setting-file")) + .templatesMetadata(new ClusterMetadataManifest.UploadedMetadataAttribute(TEMPLATES_METADATA, "mock-templates-file")) + .put(IndexGraveyard.TYPE, new ClusterMetadataManifest.UploadedMetadataAttribute(IndexGraveyard.TYPE, "mock-custom-" +IndexGraveyard.TYPE+ "-file")) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) .previousClusterUUID("prev-cluster-uuid") @@ -998,7 +978,7 @@ public void testReadGlobalMetadata() throws IOException { Metadata expactedMetadata = Metadata.builder().persistentSettings(Settings.builder().put("readonly", true).build()).build(); mockBlobContainerForGlobalMetadata(mockBlobStoreObjects(), expectedManifest, expactedMetadata); - ClusterState newClusterState = remoteClusterStateService.getLatestClusterState( + ClusterState newClusterState = remoteClusterStateService.getLatestClusterState( clusterState.getClusterName().value(), clusterState.metadata().clusterUUID() ); @@ -1036,7 +1016,7 @@ public void testReadGlobalMetadataIOException() throws IOException { BlobContainer blobContainer = mockBlobStoreObjects(); mockBlobContainerForGlobalMetadata(blobContainer, expectedManifest, expactedMetadata); - when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(globalIndexMetadataName))).thenThrow( + when(blobContainer.readBlob(RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT.blobName(globalIndexMetadataName))).thenThrow( FileNotFoundException.class ); @@ -1300,46 +1280,6 @@ public void testRemoteStateCleanupFailureStats() throws IOException { } } - public void testFileNames() { - final Index index = new Index("test-index", "index-uuid"); - final Settings idxSettings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) - .build(); - final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - String indexMetadataFileName = RemoteClusterStateService.indexMetadataFileName(indexMetadata); - String[] splittedIndexMetadataFileName = indexMetadataFileName.split(DELIMITER); - assertThat(indexMetadataFileName.split(DELIMITER).length, is(4)); - assertThat(splittedIndexMetadataFileName[0], is(METADATA_FILE_PREFIX)); - assertThat(splittedIndexMetadataFileName[1], is(RemoteStoreUtils.invertLong(indexMetadata.getVersion()))); - assertThat(splittedIndexMetadataFileName[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION))); - - verifyManifestFileNameWithCodec(MANIFEST_CURRENT_CODEC_VERSION); - verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V1); - verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V0); - } - - private void verifyManifestFileNameWithCodec(int codecVersion) { - int term = randomIntBetween(5, 10); - int version = randomIntBetween(5, 10); - String manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, true, codecVersion); - assertThat(manifestFileName.split(DELIMITER).length, is(6)); - String[] splittedName = manifestFileName.split(DELIMITER); - assertThat(splittedName[0], is(MANIFEST_FILE_PREFIX)); - assertThat(splittedName[1], is(RemoteStoreUtils.invertLong(term))); - assertThat(splittedName[2], is(RemoteStoreUtils.invertLong(version))); - assertThat(splittedName[3], is("C")); - assertThat(splittedName[5], is(String.valueOf(codecVersion))); - - manifestFileName = RemoteClusterStateService.getManifestFileName(term, version, false, codecVersion); - splittedName = manifestFileName.split(DELIMITER); - assertThat(splittedName[3], is("P")); - } - public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Exception { BlobContainer blobContainer = mock(BlobContainer.class); BlobPath blobPath = new BlobPath().add("random-path"); @@ -1370,54 +1310,6 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti assertBusy(() -> assertEquals(1, callCount.get())); } - public void testIndexMetadataUploadWaitTimeSetting() { - // verify default value - assertEquals( - RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT, - remoteClusterStateService.getIndexMetadataUploadTimeout() - ); - - // verify update index metadata upload timeout - int indexMetadataUploadTimeout = randomIntBetween(1, 10); - Settings newSettings = Settings.builder() - .put("cluster.remote_store.state.index_metadata.upload_timeout", indexMetadataUploadTimeout + "s") - .build(); - clusterSettings.applySettings(newSettings); - assertEquals(indexMetadataUploadTimeout, remoteClusterStateService.getIndexMetadataUploadTimeout().seconds()); - } - - public void testMetadataManifestUploadWaitTimeSetting() { - // verify default value - assertEquals( - RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT, - remoteClusterStateService.getMetadataManifestUploadTimeout() - ); - - // verify update metadata manifest upload timeout - int metadataManifestUploadTimeout = randomIntBetween(1, 10); - Settings newSettings = Settings.builder() - .put("cluster.remote_store.state.metadata_manifest.upload_timeout", metadataManifestUploadTimeout + "s") - .build(); - clusterSettings.applySettings(newSettings); - assertEquals(metadataManifestUploadTimeout, remoteClusterStateService.getMetadataManifestUploadTimeout().seconds()); - } - - public void testGlobalMetadataUploadWaitTimeSetting() { - // verify default value - assertEquals( - RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT, - remoteClusterStateService.getGlobalMetadataUploadTimeout() - ); - - // verify update global metadata upload timeout - int globalMetadataUploadTimeout = randomIntBetween(1, 10); - Settings newSettings = Settings.builder() - .put("cluster.remote_store.state.global_metadata.upload_timeout", globalMetadataUploadTimeout + "s") - .build(); - clusterSettings.applySettings(newSettings); - assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); - } - private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } @@ -1664,7 +1556,7 @@ private void mockBlobContainer( when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) .thenReturn(Arrays.asList(blobMetadata)); - BytesReference bytes = RemoteClusterStateService.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( + BytesReference bytes = RemoteManifestManager.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( clusterMetadataManifest, manifestFileName, blobStoreRepository.getCompressor(), @@ -1680,7 +1572,7 @@ private void mockBlobContainer( } String fileName = uploadedIndexMetadata.getUploadedFilename(); when(blobContainer.readBlob(fileName + ".dat")).thenAnswer((invocationOnMock) -> { - BytesReference bytesIndexMetadata = RemoteClusterStateService.INDEX_METADATA_FORMAT.serialize( + BytesReference bytesIndexMetadata = RemoteIndexMetadataManager.INDEX_METADATA_FORMAT.serialize( indexMetadata, fileName, blobStoreRepository.getCompressor(), @@ -1702,15 +1594,10 @@ private void mockBlobContainerForGlobalMetadata( int codecVersion = clusterMetadataManifest.getCodecVersion(); String mockManifestFileName = "manifest__1__2__C__456__" + codecVersion; BlobMetadata blobMetadata = new PlainBlobMetadata(mockManifestFileName, 1); - when( - blobContainer.listBlobsByPrefixInSortedOrder( - "manifest" + RemoteClusterStateService.DELIMITER, - 1, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ) - ).thenReturn(Arrays.asList(blobMetadata)); + when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) + .thenReturn(Arrays.asList(blobMetadata)); - BytesReference bytes = RemoteClusterStateService.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( + BytesReference bytes = RemoteManifestManager.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( clusterMetadataManifest, mockManifestFileName, blobStoreRepository.getCompressor(), @@ -1719,22 +1606,21 @@ private void mockBlobContainerForGlobalMetadata( when(blobContainer.readBlob(mockManifestFileName)).thenReturn(new ByteArrayInputStream(bytes.streamInput().readAllBytes())); if (codecVersion >= ClusterMetadataManifest.CODEC_V2) { String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); - when(blobContainer.readBlob(RemoteClusterStateService.COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))).thenAnswer( - (invocationOnMock) -> { - BytesReference bytesReference = RemoteClusterStateService.COORDINATION_METADATA_FORMAT.serialize( + when(blobContainer.readBlob(RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))) + .thenAnswer((invocationOnMock) -> { + BytesReference bytesReference = RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT.serialize( metadata.coordinationMetadata(), coordinationFileName, blobStoreRepository.getCompressor(), FORMAT_PARAMS ); return new ByteArrayInputStream(bytesReference.streamInput().readAllBytes()); - } - ); + }); String settingsFileName = getFileNameFromPath(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); - when(blobContainer.readBlob(RemoteClusterStateService.SETTINGS_METADATA_FORMAT.blobName(settingsFileName))).thenAnswer( + when(blobContainer.readBlob(RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT.blobName(settingsFileName))).thenAnswer( (invocationOnMock) -> { - BytesReference bytesReference = RemoteClusterStateService.SETTINGS_METADATA_FORMAT.serialize( + BytesReference bytesReference = RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT.serialize( metadata.persistentSettings(), settingsFileName, blobStoreRepository.getCompressor(), @@ -1745,9 +1631,9 @@ private void mockBlobContainerForGlobalMetadata( ); String templatesFileName = getFileNameFromPath(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); - when(blobContainer.readBlob(RemoteClusterStateService.TEMPLATES_METADATA_FORMAT.blobName(templatesFileName))).thenAnswer( + when(blobContainer.readBlob(RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT.blobName(templatesFileName))).thenAnswer( (invocationOnMock) -> { - BytesReference bytesReference = RemoteClusterStateService.TEMPLATES_METADATA_FORMAT.serialize( + BytesReference bytesReference = RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT.serialize( metadata.templatesMetadata(), templatesFileName, blobStoreRepository.getCompressor(), @@ -1765,9 +1651,9 @@ private void mockBlobContainerForGlobalMetadata( for (Map.Entry entry : customFileMap.entrySet()) { String custom = entry.getKey(); String fileName = entry.getValue(); - when(blobContainer.readBlob(RemoteClusterStateService.CUSTOM_METADATA_FORMAT.blobName(fileName))).thenAnswer( + when(blobContainer.readBlob(RemoteGlobalMetadataManager.CUSTOM_METADATA_FORMAT.blobName(fileName))).thenAnswer( (invocation) -> { - BytesReference bytesReference = RemoteClusterStateService.CUSTOM_METADATA_FORMAT.serialize( + BytesReference bytesReference = RemoteGlobalMetadataManager.CUSTOM_METADATA_FORMAT.serialize( metadata.custom(custom), fileName, blobStoreRepository.getCompressor(), @@ -1779,9 +1665,9 @@ private void mockBlobContainerForGlobalMetadata( } } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); - when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))) + when(blobContainer.readBlob(RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))) .thenAnswer((invocationOnMock) -> { - BytesReference bytesGlobalMetadata = RemoteClusterStateService.GLOBAL_METADATA_FORMAT.serialize( + BytesReference bytesGlobalMetadata = RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT.serialize( metadata, "global-metadata-file", blobStoreRepository.getCompressor(), @@ -1797,7 +1683,7 @@ private String getFileNameFromPath(String filePath) { return splitPath[splitPath.length - 1]; } - private static ClusterState.Builder generateClusterStateWithGlobalMetadata() { + static ClusterState.Builder generateClusterStateWithGlobalMetadata() { final Settings clusterSettings = Settings.builder().put("cluster.blocks.read_only", true).build(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); @@ -1813,7 +1699,7 @@ private static ClusterState.Builder generateClusterStateWithGlobalMetadata() { ); } - private static ClusterState.Builder generateClusterStateWithOneIndex() { + static ClusterState.Builder generateClusterStateWithOneIndex() { final Index index = new Index("test-index", "index-uuid"); final Settings idxSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -1843,7 +1729,7 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() { ); } - private static DiscoveryNodes nodesWithLocalNodeClusterManager() { + static DiscoveryNodes nodesWithLocalNodeClusterManager() { return DiscoveryNodes.builder().clusterManagerNodeId("cluster-manager-id").localNodeId("cluster-manager-id").build(); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java new file mode 100644 index 0000000000000..c9fb7d13e678a --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import static org.mockito.Mockito.mock; + +public class RemoteGlobalMetadataManagerTests extends OpenSearchTestCase { + private RemoteGlobalMetadataManager remoteGlobalMetadataManager; + private ClusterSettings clusterSettings; + private BlobStoreRepository blobStoreRepository; + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + blobStoreRepository = mock(BlobStoreRepository.class); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + } + + public void testGlobalMetadataUploadWaitTimeSetting() { + // verify default value + assertEquals( + RemoteGlobalMetadataManager.GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT, + remoteGlobalMetadataManager.getGlobalMetadataUploadTimeout() + ); + + // verify update global metadata upload timeout + int globalMetadataUploadTimeout = randomIntBetween(1, 10); + Settings newSettings = Settings.builder() + .put("cluster.remote_store.state.global_metadata.upload_timeout", globalMetadataUploadTimeout + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(globalMetadataUploadTimeout, remoteGlobalMetadataManager.getGlobalMetadataUploadTimeout().seconds()); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java new file mode 100644 index 0000000000000..8ccc34e32f4b7 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import static org.mockito.Mockito.mock; + +public class RemoteIndexMetadataManagerTests extends OpenSearchTestCase { + private RemoteIndexMetadataManager remoteIndexMetadataManager; + private BlobStoreRepository blobStoreRepository; + private ClusterSettings clusterSettings; + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + blobStoreRepository = mock(BlobStoreRepository.class); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + } + + public void testIndexMetadataUploadWaitTimeSetting() { + // verify default value + assertEquals( + RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT, + remoteIndexMetadataManager.getIndexMetadataUploadTimeout() + ); + + // verify update index metadata upload timeout + int indexMetadataUploadTimeout = randomIntBetween(1, 10); + Settings newSettings = Settings.builder() + .put("cluster.remote_store.state.index_metadata.upload_timeout", indexMetadataUploadTimeout + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(indexMetadataUploadTimeout, remoteIndexMetadataManager.getIndexMetadataUploadTimeout().seconds()); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java new file mode 100644 index 0000000000000..fb7f0612d824c --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java @@ -0,0 +1,143 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RemoteManifestManagerTests extends OpenSearchTestCase { + private RemoteManifestManager remoteManifestManager; + private ClusterSettings clusterSettings; + private BlobStoreRepository blobStoreRepository; + private BlobStore blobStore; + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + blobStoreRepository = mock(BlobStoreRepository.class); + remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, "test-node-id"); + blobStore = mock(BlobStore.class); + when(blobStoreRepository.blobStore()).thenReturn(blobStore); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + } + + public void testMetadataManifestUploadWaitTimeSetting() { + // verify default value + assertEquals( + RemoteManifestManager.METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT, + remoteManifestManager.getMetadataManifestUploadTimeout() + ); + + // verify update metadata manifest upload timeout + int metadataManifestUploadTimeout = randomIntBetween(1, 10); + Settings newSettings = Settings.builder() + .put("cluster.remote_store.state.metadata_manifest.upload_timeout", metadataManifestUploadTimeout + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(metadataManifestUploadTimeout, remoteManifestManager.getMetadataManifestUploadTimeout().seconds()); + } + + public void testFileNames() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + String indexMetadataFileName = RemoteIndexMetadataManager.indexMetadataFileName(indexMetadata); + String[] splittedIndexMetadataFileName = indexMetadataFileName.split(DELIMITER); + assertEquals(4, indexMetadataFileName.split(DELIMITER).length); + assertEquals(METADATA_FILE_PREFIX, splittedIndexMetadataFileName[0]); + assertEquals(RemoteStoreUtils.invertLong(indexMetadata.getVersion()), splittedIndexMetadataFileName[1]); + assertEquals(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION), splittedIndexMetadataFileName[3]); + + verifyManifestFileNameWithCodec(MANIFEST_CURRENT_CODEC_VERSION); + verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V1); + verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V0); + } + + private void verifyManifestFileNameWithCodec(int codecVersion) { + int term = randomIntBetween(5, 10); + int version = randomIntBetween(5, 10); + String manifestFileName = RemoteManifestManager.getManifestFileName(term, version, true, codecVersion); + assertEquals(6, manifestFileName.split(DELIMITER).length); + String[] splittedName = manifestFileName.split(DELIMITER); + assertEquals(MANIFEST_FILE_PREFIX, splittedName[0]); + assertEquals(RemoteStoreUtils.invertLong(term), splittedName[1]); + assertEquals(RemoteStoreUtils.invertLong(version), splittedName[2]); + assertEquals("C", splittedName[3]); + assertEquals(String.valueOf(codecVersion), splittedName[5]); + + manifestFileName = RemoteManifestManager.getManifestFileName(term, version, false, codecVersion); + splittedName = manifestFileName.split(DELIMITER); + assertEquals("P", splittedName[3]); + } + + public void testReadLatestMetadataManifestFailedIOException() throws IOException { + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + + BlobContainer blobContainer = mockBlobStoreObjects(); + when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) + .thenThrow(IOException.class); + + Exception e = assertThrows( + IllegalStateException.class, + () -> remoteManifestManager.getLatestClusterMetadataManifest( + clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID() + ) + ); + assertEquals(e.getMessage(), "Error while fetching latest manifest file for remote cluster state"); + } + + private BlobContainer mockBlobStoreObjects() { + final BlobPath blobPath = mock(BlobPath.class); + when((blobStoreRepository.basePath())).thenReturn(blobPath); + when(blobPath.add(anyString())).thenReturn(blobPath); + when(blobPath.buildAsString()).thenReturn("/blob/path/"); + final BlobContainer blobContainer = mock(BlobContainer.class); + when(blobContainer.path()).thenReturn(blobPath); + when(blobStore.blobContainer(any())).thenReturn(blobContainer); + return blobContainer; + } +} From 5f62745c1e6d8e884a952e5b91168d8e06241e4f Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 9 Apr 2024 10:15:59 +0530 Subject: [PATCH 014/133] Optimize stale file deletion Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 90 +++++++++++++++--- .../common/settings/ClusterSettings.java | 1 + .../remote/RemoteClusterStateService.java | 95 ++++++++++++++++++- .../RemoteClusterStateServiceTests.java | 50 ++++++++++ 4 files changed, 223 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index 4fc6550f2a3a6..d556206b8d341 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -23,11 +23,14 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; @@ -63,27 +66,62 @@ private Map initialTestSetup(int shardCount, int replicaCount, int return indexStats; } - public void testFullClusterRestoreStaleDelete() throws Exception { + public void testRemoteCleanupTaskUpdated() { int shardCount = randomIntBetween(1, 2); int replicaCount = 1; int dataNodeCount = shardCount * (replicaCount + 1); int clusterManagerNodeCount = 1; initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); - setReplicaCount(0); - setReplicaCount(2); - setReplicaCount(0); - setReplicaCount(1); - setReplicaCount(0); - setReplicaCount(1); - setReplicaCount(0); - setReplicaCount(2); - setReplicaCount(0); + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + assertEquals( + 5, + remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes() + ); + assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); + + // now disable + client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), -1)) + .get(); + assertEquals( + -1, + remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMillis() + ); + assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); + + // now set Clean up interval to 1 min + client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) + .get(); + assertEquals( + 1, + remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes() + ); + } + + public void testRemoteCleanupOnlyAfter10Updates() throws Exception { + int shardCount = randomIntBetween(1, 2); + int replicaCount = 1; + int dataNodeCount = shardCount * (replicaCount + 1); + int clusterManagerNodeCount = 1; + + initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( RemoteClusterStateService.class ); + // set cleanup interval to 1 min + client().admin().cluster().prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) + .get(); + + replicaCount = updateReplicaCountNTimes(9, replicaCount); + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); @@ -95,14 +133,30 @@ public void testFullClusterRestoreStaleDelete() throws Exception { ) .add("cluster-state") .add(getClusterState().metadata().clusterUUID()); + BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); - assertEquals(10, repository.blobStore().blobContainer(baseMetadataPath.add("manifest")).listBlobsByPrefix("manifest").size()); + assertBusy(() -> { + assertEquals(RETAINED_MANIFESTS - 1, repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size()); + }, 1, TimeUnit.MINUTES); + + replicaCount = updateReplicaCountNTimes(8, replicaCount); + + // wait for 1 min, to ensure that clean up task ran and didn't clean up stale files because it was less than 10 + Thread.sleep(60000); + assertNotEquals(RETAINED_MANIFESTS - 1, repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size()); + + // Do 2 more updates, now since the total successful state changes are more than 10, stale files will be cleaned up + replicaCount = updateReplicaCountNTimes(2, replicaCount); + + assertBusy(() -> { + assertEquals(RETAINED_MANIFESTS - 1, repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size()); + }, 1, TimeUnit.MINUTES); Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( cluster().getClusterName(), getClusterState().metadata().clusterUUID() ).getMetadata().getIndices(); - assertEquals(0, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); + assertEquals(replicaCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); } @@ -243,4 +297,16 @@ private void setReplicaCount(int replicaCount) { .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, replicaCount)) .get(); } + + private int updateReplicaCountNTimes(int n, int initialCount) { + int newReplicaCount = randomIntBetween(0, 3);; + for (int i = 0; i < n; i++) { + while (newReplicaCount == initialCount) { + newReplicaCount = randomIntBetween(0, 3); + } + setReplicaCount(newReplicaCount); + initialCount = newReplicaCount; + } + return newReplicaCount; + } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 81ee55fcc31d4..b18832e7ada41 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -706,6 +706,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, + RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteGlobalMetadataManager.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteManifestManager.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 1f7f40b8fb598..54295fe4fb183 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -15,6 +15,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; @@ -25,6 +26,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; @@ -85,6 +87,7 @@ */ public class RemoteClusterStateService implements Closeable { public static final int RETAINED_MANIFESTS = 10; + public static final int SKIP_CLEANUP_STATE_CHANGES = 10; private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); @@ -98,6 +101,17 @@ public class RemoteClusterStateService implements Closeable { Property.Final ); + /** + * Setting to specify the interval to do run stale file cleanup job + */ + public static final Setting REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING = Setting.timeSetting( + "cluster.remote_store.state.cleanup_interval", + TimeValue.timeValueMinutes(5), + TimeValue.timeValueMillis(-1), + Property.NodeScope, + Property.Dynamic + ); + private final String nodeId; private final Supplier repositoriesService; private final Settings settings; @@ -113,7 +127,13 @@ public class RemoteClusterStateService implements Closeable { private RemoteGlobalMetadataManager remoteGlobalMetadataManager; private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; + private TimeValue staleFileCleanupInterval; + private AsyncStaleFileDeletion staleFileDeletionTask; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; + private String latestClusterName; + private String latestClusterUUID; + private long lastCleanupAttemptState; + private boolean isClusterManagerNode; public RemoteClusterStateService( String nodeId, @@ -133,6 +153,10 @@ public RemoteClusterStateService( this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); this.remoteStateStats = new RemotePersistenceStats(); + this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); + clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, this::updateCleanupInterval); + this.lastCleanupAttemptState = 0; + this.isClusterManagerNode = DiscoveryNode.isClusterManagerNode(settings); } private BlobStoreTransferService getBlobStoreTransferService() { @@ -314,7 +338,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUpload || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), false ); - deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS); + this.latestClusterName = clusterState.getClusterName().value(); + this.latestClusterUUID = clusterState.metadata().clusterUUID(); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); @@ -503,6 +528,49 @@ private UploadedMetadataResults writeMetadataInParallel( return response; } + public TimeValue getStaleFileCleanupInterval() { + return this.staleFileCleanupInterval; + } + + AsyncStaleFileDeletion getStaleFileDeletionTask() { // for testing + return this.staleFileDeletionTask; + } + + private void updateCleanupInterval(TimeValue updatedInterval) { + if (!isClusterManagerNode) return; + this.staleFileCleanupInterval = updatedInterval; + logger.info("updated remote state cleanup interval to {}", updatedInterval); + // After updating the interval, we need to close the current task and create a new one which will run with updated interval + if (!this.staleFileDeletionTask.getInterval().equals(updatedInterval)) { + this.staleFileDeletionTask.setInterval(updatedInterval); + } + } + + private void cleanUpStaleFiles() { + long cleanUpAttemptState = remoteStateStats.getSuccessCount(); + if ( + cleanUpAttemptState - lastCleanupAttemptState > SKIP_CLEANUP_STATE_CHANGES && + this.latestClusterName != null && this.latestClusterUUID != null + ) { + logger.info( + "Cleaning up stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates", + this.latestClusterName, + this.latestClusterUUID, + cleanUpAttemptState - lastCleanupAttemptState + ); + deleteStaleClusterMetadata(this.latestClusterName, this.latestClusterUUID, RETAINED_MANIFESTS); + lastCleanupAttemptState = cleanUpAttemptState; + } else { + logger.info( + "Skipping cleanup of stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates, which is less than threshold {}", + this.latestClusterName, + this.latestClusterUUID, + cleanUpAttemptState - lastCleanupAttemptState, + SKIP_CLEANUP_STATE_CHANGES + ); + } + } + @Nullable public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) throws IOException { @@ -539,6 +607,9 @@ public Optional getLatestClusterMetadataManifest(String @Override public void close() throws IOException { + if (staleFileDeletionTask != null) { + staleFileDeletionTask.close(); + } if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } @@ -556,6 +627,9 @@ public void start() { remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings); remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings); remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); + if (isClusterManagerNode) { + staleFileDeletionTask = new AsyncStaleFileDeletion(this); + } } private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) { @@ -1000,4 +1074,23 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa public RemotePersistenceStats getStats() { return remoteStateStats; } + + static final class AsyncStaleFileDeletion extends AbstractAsyncTask { + private final RemoteClusterStateService remoteClusterStateService; + AsyncStaleFileDeletion(RemoteClusterStateService remoteClusterStateService) { + super(logger, remoteClusterStateService.threadpool, remoteClusterStateService.getStaleFileCleanupInterval(), true); + this.remoteClusterStateService = remoteClusterStateService; + rescheduleIfNecessary(); + } + + @Override + protected boolean mustReschedule() { + return true; + } + + @Override + protected void runInternal() { + remoteClusterStateService.cleanUpStaleFiles(); + } + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 0327ddc887f2e..4636d42fb0799 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -33,6 +33,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -84,6 +85,7 @@ import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; @@ -93,6 +95,7 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; +import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -109,6 +112,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.test.OpenSearchIntegTestCase.client; public class RemoteClusterStateServiceTests extends OpenSearchTestCase { @@ -1310,6 +1314,52 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti assertBusy(() -> assertEquals(1, callCount.get())); } + public void testRemoteCleanupTaskScheduled() { + AbstractAsyncTask cleanupTask = remoteClusterStateService.getStaleFileDeletionTask(); + assertNull(cleanupTask); + + remoteClusterStateService.start(); + assertNotNull(remoteClusterStateService.getStaleFileDeletionTask()); + assertTrue(remoteClusterStateService.getStaleFileDeletionTask().mustReschedule()); + assertEquals( + clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING), + remoteClusterStateService.getStaleFileDeletionTask().getInterval() + ); + assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); + assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isClosed()); + } + + public void testRemoteCleanupNotInitializedOnDataOnlyNode() { + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + "remote_store_repository" + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + "remote_store_repository" + ); + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .putList(NODE_ROLES_SETTING.getKey(), "d") + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + ClusterSettings dataNodeClusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + remoteClusterStateService = new RemoteClusterStateService( + "test-node-id", + repositoriesServiceSupplier, + settings, + dataNodeClusterSettings, + () -> 0L, + threadPool + ); + remoteClusterStateService.start(); + assertNull(remoteClusterStateService.getStaleFileDeletionTask()); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } From d5a17eda019297236129aa698fb3ea5c9e11faff Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Fri, 19 Apr 2024 14:36:06 +0530 Subject: [PATCH 015/133] Initial commit for RemoteRoutingTableService setup Signed-off-by: Himshikha Gupta --- .../remote/RemoteRoutingTableService.java | 104 ++++++++++++++++++ .../cluster/routing/remote/package-info.java | 10 ++ .../common/settings/ClusterSettings.java | 6 +- .../common/settings/FeatureFlagSettings.java | 3 +- .../opensearch/common/util/FeatureFlags.java | 15 ++- .../remote/RemoteClusterStateService.java | 14 +++ .../remotestore/RemoteStoreNodeAttribute.java | 20 ++++ 7 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java create mode 100644 server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java new file mode 100644 index 0000000000000..7778353a75047 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.blobstore.BlobStoreRepository; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.function.Supplier; + +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; + +/** + * A Service which provides APIs to upload and download routing table from remote store. + * + * @opensearch.internal + */ +public class RemoteRoutingTableService implements Closeable { + + /** + * Cluster setting to specify if routing table should be published to remote store + */ + public static final Setting REMOTE_ROUTING_TABLE_ENABLED_SETTING = Setting.boolSetting( + "cluster.remote_store.routing.enabled", + false, + Setting.Property.NodeScope, + Setting.Property.Final + ); + private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); + private final Settings settings; + private final Supplier repositoriesService; + private final ClusterSettings clusterSettings; + private BlobStoreRepository blobStoreRepository; + + public RemoteRoutingTableService(Supplier repositoriesService, + Settings settings, + ClusterSettings clusterSettings) { + assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; + this.repositoriesService = repositoriesService; + this.settings = settings; + this.clusterSettings = clusterSettings; + } + + public List writeFullRoutingTable(ClusterState clusterState, String previousClusterUUID) { + return null; + } + + public List writeIncrementalMetadata( + ClusterState previousClusterState, + ClusterState clusterState, + ClusterMetadataManifest previousManifest) { + return null; + } + + public RoutingTable getLatestRoutingTable(String clusterName, String clusterUUID) { + return null; + } + + public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest previousManifest, String clusterName, String clusterUUID) { + return null; + } + + private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) { + } + + @Override + public void close() throws IOException { + if (blobStoreRepository != null) { + IOUtils.close(blobStoreRepository); + } + } + + public void start() { + assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; + final String remoteStoreRepo = settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY + ); + assert remoteStoreRepo != null : "Remote routing table repository is not configured"; + final Repository repository = repositoriesService.get().repository(remoteStoreRepo); + assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; + blobStoreRepository = (BlobStoreRepository) repository; + } + +} diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java b/server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java new file mode 100644 index 0000000000000..9fe016e783f20 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Package containing class to perform operations on remote routing table */ +package org.opensearch.cluster.routing.remote; diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index dab0f6bcf1c85..0b101135a00e2 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -76,6 +76,7 @@ import org.opensearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterManagerService; import org.opensearch.cluster.service.ClusterManagerTaskThrottler; @@ -732,7 +733,10 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, - RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING + RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, + + // Remote Routing table settings + RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 985eb40711e16..255c1c87f0d89 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -36,6 +36,7 @@ protected FeatureFlagSettings( FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING, FeatureFlags.WRITEABLE_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, - FeatureFlags.PLUGGABLE_CACHE_SETTING + FeatureFlags.PLUGGABLE_CACHE_SETTING, + FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index bdfce72d106d3..abee98470f925 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -67,6 +67,11 @@ public class FeatureFlags { */ public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled"; + /** + * Gates the functionality of remote routing table. + */ + public static final String REMOTE_ROUTING_TABLE_EXPERIMENTAL = "opensearch.experimental.feature.remote_store.routing.enabled"; + public static final Setting REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_STORE_MIGRATION_EXPERIMENTAL, false, @@ -93,6 +98,13 @@ public class FeatureFlags { public static final Setting PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope); + public static final Setting REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING = Setting.boolSetting( + REMOTE_ROUTING_TABLE_EXPERIMENTAL, + false, + Property.NodeScope + ); + + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -100,7 +112,8 @@ public class FeatureFlags { TELEMETRY_SETTING, DATETIME_FORMATTER_CACHING_SETTING, WRITEABLE_REMOTE_INDEX_SETTING, - PLUGGABLE_CACHE_SETTING + PLUGGABLE_CACHE_SETTING, + REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING ); /** * Should store the settings from opensearch.yml. diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c892b475d71da..8271b1323e207 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -64,6 +65,7 @@ import java.util.stream.Collectors; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -162,6 +164,7 @@ public class RemoteClusterStateService implements Closeable { private final ThreadPool threadpool; private BlobStoreRepository blobStoreRepository; private BlobStoreTransferService blobStoreTransferService; + private RemoteRoutingTableService remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; private volatile TimeValue indexMetadataUploadTimeout; @@ -206,6 +209,11 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); + + if(isRemoteRoutingTableEnabled(settings)) { + this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, + settings, clusterSettings); + } } private BlobStoreTransferService getBlobStoreTransferService() { @@ -570,6 +578,9 @@ public void close() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } + if(this.remoteRoutingTableService != null) { + this.remoteRoutingTableService.close(); + } } public void start() { @@ -581,6 +592,9 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; + if(this.remoteRoutingTableService != null) { + this.remoteRoutingTableService.start(); + } } private ClusterMetadataManifest uploadManifest( diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index a3bfe1195d8cc..f1c2ee72a15e0 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -12,7 +12,9 @@ import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.node.Node; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -27,6 +29,8 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; + /** * This is an abstraction for validating and storing information specific to remote backed storage nodes. * @@ -45,6 +49,8 @@ public class RemoteStoreNodeAttribute { + "." + CryptoMetadata.SETTINGS_KEY; public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; + public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing.repository"; + private final RepositoriesMetadata repositoriesMetadata; /** @@ -151,6 +157,10 @@ private Set getValidatedRepositoryNames(DiscoveryNode node) { } else if (node.getAttributes().containsKey(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); } + if (node.getAttributes().containsKey(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)){ + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + } + return repositoryNames; } @@ -181,6 +191,16 @@ public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { && isRemoteClusterStateAttributePresent(settings); } + public static boolean isRemoteRoutingTableAttributePresent(Settings settings) { + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) + .isEmpty() == false; + } + + public static boolean isRemoteRoutingTableEnabled(Settings settings) { + return FeatureFlags.isEnabled(REMOTE_ROUTING_TABLE_EXPERIMENTAL) && RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) + && isRemoteRoutingTableAttributePresent(settings); + } + public RepositoriesMetadata getRepositoriesMetadata() { return this.repositoriesMetadata; } From 9176015df2852ef2cb7994f3d931f06c8d7abd14 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 22 Apr 2024 16:43:46 +0530 Subject: [PATCH 016/133] Adds unit test for remote routing setup Signed-off-by: Himshikha Gupta --- .../remote/RemoteClusterStateService.java | 5 + .../RemoteRoutingTableServiceTests.java | 92 +++++++++++++++++++ .../RemoteClusterStateServiceTests.java | 34 ++++++- 3 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 8271b1323e207..69214d6b22a61 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -741,6 +741,11 @@ public TimeValue getMetadataManifestUploadTimeout() { return this.metadataManifestUploadTimeout; } + //Package private for unit test + RemoteRoutingTableService getRemoteRoutingTableService() { + return this.remoteRoutingTableService; + } + static String getManifestFileName(long term, long version, boolean committed) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ return String.join( diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java new file mode 100644 index 0000000000000..4026a36e76600 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -0,0 +1,92 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.remote; + +import org.junit.After; +import org.junit.Before; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.repositories.FilterRepository; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.RepositoryMissingException; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.function.Supplier; + +import static org.mockito.Mockito.*; +import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; + +public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { + + private RemoteRoutingTableService remoteRoutingTableService; + private ClusterSettings clusterSettings; + private Supplier repositoriesServiceSupplier; + private RepositoriesService repositoriesService; + private BlobStoreRepository blobStoreRepository; + + @Before + public void setup() { + repositoriesServiceSupplier = mock(Supplier.class); + repositoriesService = mock(RepositoriesService.class); + when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); + + Settings settings = Settings.builder() + .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .build(); + + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + blobStoreRepository = mock(BlobStoreRepository.class); + when(repositoriesService.repository("routing_repository")).thenReturn(blobStoreRepository); + + Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + + remoteRoutingTableService = new RemoteRoutingTableService( + repositoriesServiceSupplier, + settings, + clusterSettings + ); + } + + @After + public void teardown() throws Exception { + super.tearDown(); + remoteRoutingTableService.close(); + } + + + public void testFailInitializationWhenRemoteRoutingDisabled() { + final Settings settings = Settings.builder().build(); + assertThrows( + AssertionError.class, + () -> new RemoteRoutingTableService( + repositoriesServiceSupplier, + settings, + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ) + ); + } + + public void testFailStartWhenRepositoryNotSet() { + doThrow(new RepositoryMissingException("repository missing")).when(repositoriesService).repository("routing_repository"); + assertThrows(RepositoryMissingException.class, () -> remoteRoutingTableService.start()); + } + + public void testFailStartWhenNotBlobRepository() { + final FilterRepository filterRepository = mock(FilterRepository.class); + when(repositoriesService.repository("routing_repository")).thenReturn(filterRepository); + assertThrows(AssertionError.class, () -> remoteRoutingTableService.start()); + } + +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 65477051cdb30..78aa1173bea62 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -31,6 +32,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -77,6 +79,7 @@ import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; +import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; @@ -84,9 +87,6 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -99,6 +99,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.*; public class RemoteClusterStateServiceTests extends OpenSearchTestCase { @@ -1218,6 +1219,33 @@ public void testGlobalMetadataUploadWaitTimeSetting() { assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); } + public void testRemoteRoutingTableNotInitializedWhenDisabled() { + assertNull(remoteClusterStateService.getRemoteRoutingTableService()); + } + + public void testRemoteRoutingTableInitializedWhenEnabled() { + Settings newSettings = Settings.builder() + .put(RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.getKey(), true) + .put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, "routing_repository") + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + clusterSettings.applySettings(newSettings); + + Settings nodeSettings = Settings.builder().put(REMOTE_ROUTING_TABLE_EXPERIMENTAL, "true").build(); + FeatureFlags.initializeFeatureFlags(nodeSettings); + + remoteClusterStateService = new RemoteClusterStateService( + "test-node-id", + repositoriesServiceSupplier, + newSettings, + clusterSettings, + () -> 0L, + threadPool + ); + assertNotNull(remoteClusterStateService.getRemoteRoutingTableService()); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } From a17cf0622cfd267c391d79ff3c72af30ec249b59 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Wed, 17 Apr 2024 14:07:08 +0530 Subject: [PATCH 017/133] Initial commit for index routing table manifest Signed-off-by: Bukhtawar Khan --- .../remote/ClusterMetadataManifest.java | 98 ++++++++++++++++++- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4725f40076ce2..4e5891d154de0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -35,6 +35,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. + public static final int CODEC_V2 = 2; // In Codec V2 we introduce index routing-metadata in manifest file. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -48,6 +49,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); private static long term(Object[] fields) { return (long) fields[0]; @@ -97,6 +99,10 @@ private static String globalMetadataFileName(Object[] fields) { return (String) fields[11]; } + private static List indicesRouting(Object[] fields) { + return (List) fields[12]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> new ClusterMetadataManifest( @@ -133,11 +139,31 @@ private static String globalMetadataFileName(Object[] fields) { ) ); - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V1; + private static final ConstructingObjectParser PARSER_V2 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> new ClusterMetadataManifest( + term(fields), + version(fields), + clusterUUID(fields), + stateUUID(fields), + opensearchVersion(fields), + nodeId(fields), + committed(fields), + codecVersion(fields), + globalMetadataFileName(fields), + indices(fields), + previousClusterUUID(fields), + clusterUUIDCommitted(fields), + indicesRouting(fields) + ) + ); + + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); + declareParser(PARSER_V2, CODEC_V2); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -160,6 +186,13 @@ private static void declareParser(ConstructingObjectParser= CODEC_V2) { + parser.declareObjectArray( + ConstructingObjectParser.constructorArg(), + (p, c) -> UploadedIndexMetadata.fromXContent(p), + INDICES_ROUTING_FIELD + ); + } } private final int codecVersion; @@ -174,6 +207,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; public List getIndices() { return indices; @@ -223,6 +257,10 @@ public String getGlobalMetadataFileName() { return globalMetadataFileName; } + public List getIndicesRouting() { + return indicesRouting; + } + public ClusterMetadataManifest( long clusterTerm, long version, @@ -237,6 +275,25 @@ public ClusterMetadataManifest( String previousClusterUUID, boolean clusterUUIDCommitted ) { + this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, null); + } + + public ClusterMetadataManifest( + long clusterTerm, + long version, + String clusterUUID, + String stateUUID, + Version opensearchVersion, + String nodeId, + boolean committed, + int codecVersion, + String globalMetadataFileName, + List indices, + String previousClusterUUID, + boolean clusterUUIDCommitted, + List indicesRouting + ) { this.clusterTerm = clusterTerm; this.stateVersion = version; this.clusterUUID = clusterUUID; @@ -249,6 +306,7 @@ public ClusterMetadataManifest( this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; + this.indicesRouting = Collections.unmodifiableList(indicesRouting); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -262,12 +320,18 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_2_12_0)) { + if (in.getVersion().onOrAfter(Version.V_2_14_0)) { + this.codecVersion = in.readInt(); + this.globalMetadataFileName = in.readString(); + this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); + } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.indicesRouting = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; + this.indicesRouting = null; } } @@ -301,6 +365,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } + if (onOrAfterCodecVersion(CODEC_V2)) { + builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); + { + for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { + uploadedIndexMetadata.toXContent(builder, params); + } + } + } return builder; } @@ -319,6 +391,8 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); + } else if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + out.writeCollection(indicesRouting); } } @@ -342,7 +416,8 @@ public boolean equals(Object o) { && Objects.equals(previousClusterUUID, that.previousClusterUUID) && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) - && Objects.equals(codecVersion, that.codecVersion); + && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(indicesRouting, that.indicesRouting); } @Override @@ -359,7 +434,8 @@ public int hashCode() { nodeId, committed, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + indicesRouting ); } @@ -399,12 +475,18 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; + private List indicesRouting; public Builder indices(List indices) { this.indices = indices; return this; } + public Builder indicesRouting(List indicesRouting) { + this.indicesRouting = indicesRouting; + return this; + } + public Builder codecVersion(int codecVersion) { this.codecVersion = codecVersion; return this; @@ -454,6 +536,10 @@ public List getIndices() { return indices; } + public List getIndicesRouting() { + return indicesRouting; + } + public Builder previousClusterUUID(String previousClusterUUID) { this.previousClusterUUID = previousClusterUUID; return this; @@ -481,6 +567,7 @@ public Builder(ClusterMetadataManifest manifest) { this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; + this.indicesRouting = new ArrayList<>(manifest.indicesRouting); } public ClusterMetadataManifest build() { @@ -496,7 +583,8 @@ public ClusterMetadataManifest build() { globalMetadataFileName, indices, previousClusterUUID, - clusterUUIDCommitted + clusterUUIDCommitted, + indicesRouting ); } From 3dd93b4ec76bcb3a5e110316cd4fa3dabf0f1826 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:00:39 +0530 Subject: [PATCH 018/133] Changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan --- .../stream}/BufferedChecksumStreamInput.java | 2 +- .../stream}/BufferedChecksumStreamOutput.java | 2 +- .../org/opensearch/common/util/BigArrays.java | 2 +- .../routingtable/IndexRoutingTableHeader.java | 135 ++++++++++++++++++ .../index/translog/BaseTranslogReader.java | 1 + .../opensearch/index/translog/Translog.java | 2 + .../index/translog/TranslogHeader.java | 2 + .../index/translog/TranslogSnapshot.java | 1 + .../index/translog/TranslogWriter.java | 1 + .../coordination/CoordinationStateTests.java | 2 +- .../remote/ClusterMetadataManifestTests.java | 6 +- .../snapshots/BlobStoreFormatTests.java | 2 +- 12 files changed, 150 insertions(+), 8 deletions(-) rename server/src/main/java/org/opensearch/{index/translog => common/io/stream}/BufferedChecksumStreamInput.java (99%) rename server/src/main/java/org/opensearch/{index/translog => common/io/stream}/BufferedChecksumStreamOutput.java (98%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java similarity index 99% rename from server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java rename to server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java index f75f27b7bcb91..f3341712275f9 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamInput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamInput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.translog; +package org.opensearch.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.apache.lucene.util.BitUtil; diff --git a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java similarity index 98% rename from server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java rename to server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java index 9e96664c79cc5..254f228f1c739 100644 --- a/server/src/main/java/org/opensearch/index/translog/BufferedChecksumStreamOutput.java +++ b/server/src/main/java/org/opensearch/common/io/stream/BufferedChecksumStreamOutput.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.index.translog; +package org.opensearch.common.io.stream; import org.apache.lucene.store.BufferedChecksum; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 92371c2c77ef9..734e5535c3cf4 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -49,7 +49,7 @@ import java.util.Arrays; /** - * Utility class to work with arrays. + * Utility class to work with arrays.Ø * * @opensearch.api * */ diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java new file mode 100644 index 0000000000000..87d94dc1147d1 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -0,0 +1,135 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.store.InputStreamDataInput; +import org.apache.lucene.store.OutputStreamDataOutput; +import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; + +import java.io.EOFException; +import java.io.IOException; + +/** + * The stored header information for the individual index routing table + */ +public class IndexRoutingTableHeader { + + private int routingTableVersion; + + private String indexName; + + private Version nodeVersion; + + public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; + + public static final int INITIAL_VERSION = 1; + + public static final int CURRENT_VERSION = INITIAL_VERSION; + + + public IndexRoutingTableHeader(int routingTableVersion, String indexName, Version nodeVersion) { + this.routingTableVersion = routingTableVersion; + this.indexName = indexName; + this.nodeVersion = nodeVersion; + } + + /** + * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * @return the {@link BytesReference} + * @throws IOException + */ + public BytesReference write() throws IOException { + BytesReference bytesReference; + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput)) { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + // Write version + out.writeInt(routingTableVersion); + out.writeInt(nodeVersion.id); + out.writeString(indexName); + // Checksum header + out.writeInt((int) out.getChecksum()); + out.flush(); + bytesReference = bytesStreamOutput.bytes(); + } + return bytesReference; + } + + + /** + * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * @param inBytes + * @param source + * @return + * @throws IOException + */ + public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { + try { + try(BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { + readHeaderVersion(in); + final int version = in.readInt(); + final int nodeVersion = in.readInt(); + final String name = in.readString(); + verifyChecksum(in); + assert version >= 0 : "Version must be non-negative [" + version + "]"; + assert in.readByte() == -1 : "Header is not fully read"; + return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); + } + } catch (EOFException e) { + throw new IOException("index routing header truncated", e); + } + } + + + static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { + // This absolutely must come first, or else reading the checksum becomes part of the checksum + long expectedChecksum = in.getChecksum(); + long readChecksum = Integer.toUnsignedLong(in.readInt()); + if (readChecksum != expectedChecksum) { + throw new IOException( + "checksum verification failed - expected: 0x" + + Long.toHexString(expectedChecksum) + + ", got: 0x" + + Long.toHexString(readChecksum) + ); + } + } + + static int readHeaderVersion(final StreamInput in) throws IOException { + final int version; + try { + version = CodecUtil.checkHeader(new InputStreamDataInput(in), INDEX_ROUTING_HEADER_CODEC, INITIAL_VERSION, CURRENT_VERSION); + } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) { + throw new IOException("index routing table header corrupted", e); + } + return version; + } + + public int getRoutingTableVersion() { + return routingTableVersion; + } + + public String getIndexName() { + return indexName; + } + + public Version getNodeVersion() { + return nodeVersion; + } +} diff --git a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java index d6fa2a2e53de3..9088eb6b20fb8 100644 --- a/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java +++ b/server/src/main/java/org/opensearch/index/translog/BaseTranslogReader.java @@ -32,6 +32,7 @@ package org.opensearch.index.translog; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.core.common.io.stream.ByteBufferStreamInput; import org.opensearch.index.seqno.SequenceNumbers; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index c653605f8fa10..905abb65ccc2c 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -38,6 +38,8 @@ import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java index 7b5be9505f27a..c622214a3a69d 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogHeader.java @@ -40,6 +40,8 @@ import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.util.BytesRef; import org.opensearch.common.io.Channels; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java index 89718156cbbe8..a6e322bf58fff 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogSnapshot.java @@ -32,6 +32,7 @@ package org.opensearch.index.translog; import org.opensearch.common.io.Channels; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.index.seqno.SequenceNumbers; import java.io.EOFException; diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java index 86f7567f3333d..da3c7a8dee219 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogWriter.java @@ -42,6 +42,7 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.Channels; import org.opensearch.common.io.DiskIoBufferPool; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; import org.opensearch.common.io.stream.ReleasableBytesStreamOutput; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 1c0dc7fc1ca2d..0fe52d2e72d4b 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -942,7 +942,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep randomAlphaOfLength(10), Collections.emptyList(), randomAlphaOfLength(10), - true + true, ); Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6c9a3201656d7..647f4c913931f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -40,7 +40,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true + true, ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -67,7 +67,7 @@ public void testClusterMetadataManifestXContent() throws IOException { "test-global-metadata-file", Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true + true, ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -93,7 +93,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "test-global-metadata-file", randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", - true + true, ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( diff --git a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java index c5f36fcc01983..8c64f9a3170b2 100644 --- a/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java +++ b/server/src/test/java/org/opensearch/snapshots/BlobStoreFormatTests.java @@ -55,7 +55,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.translog.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.test.OpenSearchTestCase; From 933cdc25ec343337ccb2db6fbe7531ba9736e393 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sat, 20 Apr 2024 00:04:24 +0530 Subject: [PATCH 019/133] Revert unintentional changes for IndexRoutingTableHeader Signed-off-by: Bukhtawar Khan --- .../src/main/java/org/opensearch/common/util/BigArrays.java | 2 +- .../cluster/coordination/CoordinationStateTests.java | 2 +- .../gateway/remote/ClusterMetadataManifestTests.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/util/BigArrays.java b/server/src/main/java/org/opensearch/common/util/BigArrays.java index 734e5535c3cf4..92371c2c77ef9 100644 --- a/server/src/main/java/org/opensearch/common/util/BigArrays.java +++ b/server/src/main/java/org/opensearch/common/util/BigArrays.java @@ -49,7 +49,7 @@ import java.util.Arrays; /** - * Utility class to work with arrays.Ø + * Utility class to work with arrays. * * @opensearch.api * */ diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index 0fe52d2e72d4b..1c0dc7fc1ca2d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -942,7 +942,7 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep randomAlphaOfLength(10), Collections.emptyList(), randomAlphaOfLength(10), - true, + true ); Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 647f4c913931f..6c9a3201656d7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -40,7 +40,7 @@ public void testClusterMetadataManifestXContentV0() throws IOException { null, Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true, + true ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -67,7 +67,7 @@ public void testClusterMetadataManifestXContent() throws IOException { "test-global-metadata-file", Collections.singletonList(uploadedIndexMetadata), "prev-cluster-uuid", - true, + true ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -93,7 +93,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "test-global-metadata-file", randomUploadedIndexMetadataList(), "yfObdx8KSMKKrXf8UyHhM", - true, + true ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( From f35f0dc3a63b4199414a6926848fccd7f6d890e0 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Sun, 21 Apr 2024 00:15:47 +0530 Subject: [PATCH 020/133] Changes for IndexRoutingTableInputStream Signed-off-by: Bukhtawar Khan --- .../routingtable/IndexRoutingTableHeader.java | 27 ++- .../IndexRoutingTableInputStream.java | 157 ++++++++++++++++++ 2 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 87d94dc1147d1..04a0f1868b64f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -15,12 +15,12 @@ import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; import org.opensearch.Version; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import java.io.EOFException; import java.io.IOException; @@ -30,11 +30,11 @@ */ public class IndexRoutingTableHeader { - private int routingTableVersion; + private final long routingTableVersion; - private String indexName; + private final String indexName; - private Version nodeVersion; + private final Version nodeVersion; public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; @@ -42,8 +42,7 @@ public class IndexRoutingTableHeader { public static final int CURRENT_VERSION = INITIAL_VERSION; - - public IndexRoutingTableHeader(int routingTableVersion, String indexName, Version nodeVersion) { + public IndexRoutingTableHeader(long routingTableVersion, String indexName, Version nodeVersion) { this.routingTableVersion = routingTableVersion; this.indexName = indexName; this.nodeVersion = nodeVersion; @@ -56,11 +55,13 @@ public IndexRoutingTableHeader(int routingTableVersion, String indexName, Versio */ public BytesReference write() throws IOException { BytesReference bytesReference; - try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput)) { + try ( + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) + ) { CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); // Write version - out.writeInt(routingTableVersion); + out.writeLong(routingTableVersion); out.writeInt(nodeVersion.id); out.writeString(indexName); // Checksum header @@ -71,7 +72,6 @@ public BytesReference write() throws IOException { return bytesReference; } - /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} * @param inBytes @@ -81,7 +81,7 @@ public BytesReference write() throws IOException { */ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { try { - try(BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); final int version = in.readInt(); final int nodeVersion = in.readInt(); @@ -96,7 +96,6 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce } } - static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { // This absolutely must come first, or else reading the checksum becomes part of the checksum long expectedChecksum = in.getChecksum(); @@ -121,7 +120,7 @@ static int readHeaderVersion(final StreamInput in) throws IOException { return version; } - public int getRoutingTableVersion() { + public long getRoutingTableVersion() { return routingTableVersion; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java new file mode 100644 index 0000000000000..ac65232e9a24d --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.opensearch.Version; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.bytes.BytesReference; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; + +public class IndexRoutingTableInputStream extends InputStream { + + /** + * The buffer where data is stored. + */ + protected byte[] buf; + + /** + * The number of valid bytes in the buffer. + */ + protected int count; + + /** + * The buffer left over from the last fill + */ + protected byte[] leftOverBuf; + + /** + * The mark position + */ + protected int markPos = -1; + + /** + * The read limit + */ + protected int markLimit; + + /** + * The position + */ + protected int pos; + + private static final int BUFFER_SIZE = 8192; + + private final IndexRoutingTableHeader indexRoutingTableHeader; + + private final Iterator shardIter; + + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { + this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); + } + + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion, int size) + throws IOException { + this.buf = new byte[size]; + this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + initialFill(); + } + + @Override + public int read() throws IOException { + if (pos >= count) { + maybeResizeAndFill(); + if (pos >= count) return -1; + } + return buf[pos++] & 0xff; + } + + private void initialFill() throws IOException { + BytesReference bytesReference = indexRoutingTableHeader.write(); + buf = bytesReference.toBytesRef().bytes; + count = bytesReference.length(); + fill(buf); + } + + private void fill(byte[] buf) throws IOException { + if (leftOverBuf != null) { + System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + } + if (count < buf.length && shardIter.hasNext()) { + IndexShardRoutingTable next = shardIter.next(); + BytesReference bytesRef; + try ( + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) + ) { + IndexShardRoutingTable.Builder.writeTo(next, out); + // Checksum header + out.writeInt((int) out.getChecksum()); + out.flush(); + bytesRef = bytesStreamOutput.bytes(); + } + if (bytesRef.length() < buf.length - count) { + System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); + count += bytesRef.length(); + leftOverBuf = null; + } else { + System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); + count += buf.length - count; + leftOverBuf = new byte[bytesRef.length() - count]; + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count + 1, leftOverBuf, 0, bytesRef.length() - count); + } + } + } + + private void maybeResizeAndFill() throws IOException { + byte[] buffer = buf; + if (markPos == -1) pos = 0; /* no mark: throw away the buffer */ + else if (pos >= buffer.length) { /* no room left in buffer */ + if (markPos > 0) { /* can throw away early part of the buffer */ + int sz = pos - markPos; + System.arraycopy(buffer, markPos, buffer, 0, sz); + pos = sz; + markPos = 0; + } else if (buffer.length >= markLimit) { + markPos = -1; /* buffer got too big, invalidate mark */ + pos = 0; /* drop buffer contents */ + } else { /* grow buffer */ + int nsz = markLimit + 1; + byte[] nbuf = new byte[nsz]; + System.arraycopy(buffer, 0, nbuf, 0, pos); + buffer = nbuf; + } + } + count = pos; + fill(buffer); + } + + @Override + public void mark(int readlimit) { + markLimit = readlimit; + markPos = pos; + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void reset() throws IOException { + if (markPos < 0) throw new IOException("Resetting to invalid mark"); + pos = markPos; + } +} From 8efc1edd9bc96ad279d171e6e7dbbe637395fd34 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 29 Apr 2024 15:51:43 +0530 Subject: [PATCH 021/133] fix build after merge Signed-off-by: Shivansh Arora --- .../cluster/metadata/TemplatesMetadata.java | 3 +++ .../remote/RemoteClusterStateService.java | 20 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java index 84c62c22799e8..4d5cb4da3a65f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java @@ -9,6 +9,7 @@ package org.opensearch.cluster.metadata; import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; @@ -25,6 +26,7 @@ * * @opensearch.internal */ +@PublicApi(since = "3.0.0") public class TemplatesMetadata extends AbstractDiffable implements ToXContentFragment { public static TemplatesMetadata EMPTY_METADATA = builder().build(); private final Map templates; @@ -83,6 +85,7 @@ public int hashCode() { * * @opensearch.api */ + @PublicApi(since = "3.0.0") public static class Builder { private final Map templates; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 9f874fcdfe8a1..00d5f1751143e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -272,9 +272,11 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri return null; } + List indexMetadataList = new ArrayList<>(clusterState.metadata().indices().values()); UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel( clusterState, - new ArrayList<>(clusterState.metadata().indices().values()), + indexMetadataList, + ClusterState.UNKNOWN_UUID.equals(previousClusterUUID) ? indexMetadataList : Collections.emptyList(), clusterState.metadata().customs(), true, true, @@ -391,11 +393,20 @@ public ClusterMetadataManifest writeIncrementalMetadata( // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, // If file is empty and codec is 1 then write global metadata. if (firstUpload) { - uploadedMetadataResults = writeMetadataInParallel(clusterState, toUpload, clusterState.metadata().customs(), true, true, true); + uploadedMetadataResults = writeMetadataInParallel( + clusterState, + toUpload, + newIndexMetadataList, + clusterState.metadata().customs(), + true, + true, + true + ); } else { uploadedMetadataResults = writeMetadataInParallel( clusterState, toUpload, + newIndexMetadataList, customsToUpload, updateCoordinationMetadata, updateSettingsMetadata, @@ -473,8 +484,9 @@ private UploadedMetadataResults writeMetadataInParallel( boolean uploadSettingsMetadata, boolean uploadTemplateMetadata ) throws IOException { - int totalUploadTasks = indexToUpload.size() + newIndexMetadataList.size() + customToUpload.size() + - (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0); + int totalUploadTasks = indexToUpload.size() + newIndexMetadataList.size() + customToUpload.size() + (uploadCoordinationMetadata + ? 1 + : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new HashMap<>(totalUploadTasks); Map results = new HashMap<>(totalUploadTasks); From 55ed2451970dbbce6387877384d7423c6c4a6a89 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 9 Apr 2024 10:15:59 +0530 Subject: [PATCH 022/133] Optimize remote state stale file deletion Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 97 ++++++++++++++++--- .../common/settings/ClusterSettings.java | 1 + .../remote/RemoteClusterStateService.java | 95 +++++++++++++++++- .../RemoteClusterStateServiceTests.java | 49 ++++++++++ 4 files changed, 229 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index dfde1b958882c..fd9a71d77fd1b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -22,10 +22,13 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase { @@ -55,27 +58,59 @@ private Map initialTestSetup(int shardCount, int replicaCount, int return indexStats; } - public void testFullClusterRestoreStaleDelete() throws Exception { + public void testRemoteCleanupTaskUpdated() { int shardCount = randomIntBetween(1, 2); int replicaCount = 1; int dataNodeCount = shardCount * (replicaCount + 1); int clusterManagerNodeCount = 1; initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); - setReplicaCount(0); - setReplicaCount(2); - setReplicaCount(0); - setReplicaCount(1); - setReplicaCount(0); - setReplicaCount(1); - setReplicaCount(0); - setReplicaCount(2); - setReplicaCount(0); + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + + assertEquals(5, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); + assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); + + // now disable + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), -1)) + .get(); + + assertEquals(-1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMillis()); + assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); + // now set Clean up interval to 1 min + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) + .get(); + assertEquals(1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); + } + + public void testRemoteCleanupOnlyAfter10Updates() throws Exception { + int shardCount = randomIntBetween(1, 2); + int replicaCount = 1; + int dataNodeCount = shardCount * (replicaCount + 1); + int clusterManagerNodeCount = 1; + + initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( RemoteClusterStateService.class ); + // set cleanup interval to 1 min + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) + .get(); + + replicaCount = updateReplicaCountNTimes(9, replicaCount); + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); @@ -87,14 +122,39 @@ public void testFullClusterRestoreStaleDelete() throws Exception { ) .add("cluster-state") .add(getClusterState().metadata().clusterUUID()); + BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); - assertEquals(10, repository.blobStore().blobContainer(baseMetadataPath.add("manifest")).listBlobsByPrefix("manifest").size()); + assertBusy(() -> { + assertEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); + }, 1, TimeUnit.MINUTES); + + replicaCount = updateReplicaCountNTimes(8, replicaCount); + + // wait for 1 min, to ensure that clean up task ran and didn't clean up stale files because it was less than 10 + Thread.sleep(60000); + assertNotEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); + + // Do 2 more updates, now since the total successful state changes are more than 10, stale files will be cleaned up + replicaCount = updateReplicaCountNTimes(2, replicaCount); + + assertBusy(() -> { + assertEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); + }, 1, TimeUnit.MINUTES); Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( cluster().getClusterName(), getClusterState().metadata().clusterUUID() ).getMetadata().getIndices(); - assertEquals(0, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); + assertEquals(replicaCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); } @@ -196,4 +256,17 @@ private void setReplicaCount(int replicaCount) { .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, replicaCount)) .get(); } + + private int updateReplicaCountNTimes(int n, int initialCount) { + int newReplicaCount = randomIntBetween(0, 3); + ; + for (int i = 0; i < n; i++) { + while (newReplicaCount == initialCount) { + newReplicaCount = randomIntBetween(0, 3); + } + setReplicaCount(newReplicaCount); + initialCount = newReplicaCount; + } + return newReplicaCount; + } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 4a5a45eb1a17a..2e82314106492 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -712,6 +712,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, + RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index eaf607564185c..c0c5cf08148cb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -25,6 +26,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; @@ -78,6 +80,7 @@ public class RemoteClusterStateService implements Closeable { public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; public static final int RETAINED_MANIFESTS = 10; + public static final int SKIP_CLEANUP_STATE_CHANGES = 10; public static final String DELIMITER = "__"; @@ -147,6 +150,17 @@ public class RemoteClusterStateService implements Closeable { Property.Final ); + /** + * Setting to specify the interval to do run stale file cleanup job + */ + public static final Setting REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING = Setting.timeSetting( + "cluster.remote_store.state.cleanup_interval", + TimeValue.timeValueMinutes(5), + TimeValue.timeValueMillis(-1), + Property.NodeScope, + Property.Dynamic + ); + public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; public static final String INDEX_PATH_TOKEN = "index"; public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; @@ -169,11 +183,17 @@ public class RemoteClusterStateService implements Closeable { private volatile TimeValue globalMetadataUploadTimeout; private volatile TimeValue metadataManifestUploadTimeout; + private TimeValue staleFileCleanupInterval; + private AsyncStaleFileDeletion staleFileDeletionTask; private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; + private String latestClusterName; + private String latestClusterUUID; + private long lastCleanupAttemptState; + private boolean isClusterManagerNode; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. @@ -209,6 +229,10 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); + this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); + clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, this::updateCleanupInterval); + this.lastCleanupAttemptState = 0; + this.isClusterManagerNode = DiscoveryNode.isClusterManagerNode(settings); this.indexMetadataUploadListeners = indexMetadataUploadListeners; } @@ -356,7 +380,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( globalMetadataFile, false ); - deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS); + this.latestClusterName = clusterState.getClusterName().value(); + this.latestClusterUUID = clusterState.metadata().clusterUUID(); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); @@ -619,6 +644,48 @@ private void writeIndexMetadataAsync( ); } + public TimeValue getStaleFileCleanupInterval() { + return this.staleFileCleanupInterval; + } + + AsyncStaleFileDeletion getStaleFileDeletionTask() { // for testing + return this.staleFileDeletionTask; + } + + private void updateCleanupInterval(TimeValue updatedInterval) { + if (!isClusterManagerNode) return; + this.staleFileCleanupInterval = updatedInterval; + logger.info("updated remote state cleanup interval to {}", updatedInterval); + // After updating the interval, we need to close the current task and create a new one which will run with updated interval + if (!this.staleFileDeletionTask.getInterval().equals(updatedInterval)) { + this.staleFileDeletionTask.setInterval(updatedInterval); + } + } + + private void cleanUpStaleFiles() { + long cleanUpAttemptState = remoteStateStats.getSuccessCount(); + if (cleanUpAttemptState - lastCleanupAttemptState > SKIP_CLEANUP_STATE_CHANGES + && this.latestClusterName != null + && this.latestClusterUUID != null) { + logger.info( + "Cleaning up stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates", + this.latestClusterName, + this.latestClusterUUID, + cleanUpAttemptState - lastCleanupAttemptState + ); + deleteStaleClusterMetadata(this.latestClusterName, this.latestClusterUUID, RETAINED_MANIFESTS); + lastCleanupAttemptState = cleanUpAttemptState; + } else { + logger.info( + "Skipping cleanup of stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates, which is less than threshold {}", + this.latestClusterName, + this.latestClusterUUID, + cleanUpAttemptState - lastCleanupAttemptState, + SKIP_CLEANUP_STATE_CHANGES + ); + } + } + @Nullable public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) throws IOException { @@ -641,6 +708,9 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat @Override public void close() throws IOException { + if (staleFileDeletionTask != null) { + staleFileDeletionTask.close(); + } if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } @@ -655,6 +725,9 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; + if (isClusterManagerNode) { + staleFileDeletionTask = new AsyncStaleFileDeletion(this); + } } private ClusterMetadataManifest uploadManifest( @@ -1397,4 +1470,24 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa public RemotePersistenceStats getStats() { return remoteStateStats; } + + static final class AsyncStaleFileDeletion extends AbstractAsyncTask { + private final RemoteClusterStateService remoteClusterStateService; + + AsyncStaleFileDeletion(RemoteClusterStateService remoteClusterStateService) { + super(logger, remoteClusterStateService.threadpool, remoteClusterStateService.getStaleFileCleanupInterval(), true); + this.remoteClusterStateService = remoteClusterStateService; + rescheduleIfNecessary(); + } + + @Override + protected boolean mustReschedule() { + return true; + } + + @Override + protected void runInternal() { + remoteClusterStateService.cleanUpStaleFiles(); + } + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 9f321cd62847c..194d22584dcfd 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -31,6 +31,7 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -84,7 +85,9 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; +import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -1222,6 +1225,52 @@ public void testGlobalMetadataUploadWaitTimeSetting() { assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); } + public void testRemoteCleanupTaskScheduled() { + AbstractAsyncTask cleanupTask = remoteClusterStateService.getStaleFileDeletionTask(); + assertNull(cleanupTask); + + remoteClusterStateService.start(); + assertNotNull(remoteClusterStateService.getStaleFileDeletionTask()); + assertTrue(remoteClusterStateService.getStaleFileDeletionTask().mustReschedule()); + assertEquals( + clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING), + remoteClusterStateService.getStaleFileDeletionTask().getInterval() + ); + assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); + assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isClosed()); + } + + public void testRemoteCleanupNotInitializedOnDataOnlyNode() { + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + "remote_store_repository" + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + "remote_store_repository" + ); + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .putList(NODE_ROLES_SETTING.getKey(), "d") + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + ClusterSettings dataNodeClusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + remoteClusterStateService = new RemoteClusterStateService( + "test-node-id", + repositoriesServiceSupplier, + settings, + dataNodeClusterSettings, + () -> 0L, + threadPool + ); + remoteClusterStateService.start(); + assertNull(remoteClusterStateService.getStaleFileDeletionTask()); + } + private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } From bef9d2e5ea42d0cee829a38cec67d53787d89b28 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 12 Apr 2024 00:25:21 +0530 Subject: [PATCH 023/133] Added UT Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateService.java | 10 +++++---- .../RemoteClusterStateServiceTests.java | 22 ++++++++++++++++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c0c5cf08148cb..c0b63fe8f93a8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -91,6 +91,8 @@ public class RemoteClusterStateService implements Closeable { public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); + public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT = TimeValue.timeValueMillis(60000); + public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM = TimeValue.MINUS_ONE; public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( "cluster.remote_store.state.index_metadata.upload_timeout", @@ -155,8 +157,8 @@ public class RemoteClusterStateService implements Closeable { */ public static final Setting REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING = Setting.timeSetting( "cluster.remote_store.state.cleanup_interval", - TimeValue.timeValueMinutes(5), - TimeValue.timeValueMillis(-1), + CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, + CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM, Property.NodeScope, Property.Dynamic ); @@ -657,8 +659,8 @@ private void updateCleanupInterval(TimeValue updatedInterval) { this.staleFileCleanupInterval = updatedInterval; logger.info("updated remote state cleanup interval to {}", updatedInterval); // After updating the interval, we need to close the current task and create a new one which will run with updated interval - if (!this.staleFileDeletionTask.getInterval().equals(updatedInterval)) { - this.staleFileDeletionTask.setInterval(updatedInterval); + if (staleFileDeletionTask != null && !staleFileDeletionTask.getInterval().equals(updatedInterval)) { + staleFileDeletionTask.setInterval(updatedInterval); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 194d22584dcfd..bf03b99eb2109 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -41,6 +41,7 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; @@ -79,6 +80,8 @@ import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.opensearch.gateway.remote.RemoteClusterStateService.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; @@ -1225,10 +1228,27 @@ public void testGlobalMetadataUploadWaitTimeSetting() { assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); } + public void testRemoteClusterStateCleanupSetting() { + remoteClusterStateService.start(); + // verify default value + assertEquals( + CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, + remoteClusterStateService.getStaleFileCleanupInterval() + ); + + // verify update interval + int cleanupInterval = randomIntBetween(1, 10); + Settings newSettings = Settings.builder() + .put("cluster.remote_store.state.cleanup_interval", cleanupInterval + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(cleanupInterval, remoteClusterStateService.getStaleFileCleanupInterval().seconds()); + } + public void testRemoteCleanupTaskScheduled() { AbstractAsyncTask cleanupTask = remoteClusterStateService.getStaleFileDeletionTask(); assertNull(cleanupTask); - + // now the task should be initialized remoteClusterStateService.start(); assertNotNull(remoteClusterStateService.getStaleFileDeletionTask()); assertTrue(remoteClusterStateService.getStaleFileDeletionTask().mustReschedule()); From 6bed64b532a2e83f1ffce4ca37689e79d45db94c Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 22 Apr 2024 11:44:36 +0530 Subject: [PATCH 024/133] Refactored into a clean up manager file Signed-off-by: Shivansh Arora --- .../RemoteClusterStateCleanupManagerIT.java | 174 +++++++++ .../remote/RemoteClusterStateServiceIT.java | 148 ------- .../RemoteStoreBaseIntegTestCase.java | 10 + .../common/settings/ClusterSettings.java | 3 +- .../RemoteClusterStateCleanupManager.java | 365 ++++++++++++++++++ .../remote/RemoteClusterStateService.java | 322 ++------------- .../main/java/org/opensearch/node/Node.java | 7 +- .../GatewayMetaStatePersistedStateTests.java | 4 +- ...RemoteClusterStateCleanupManagerTests.java | 335 ++++++++++++++++ .../RemoteClusterStateServiceTests.java | 206 +--------- 10 files changed, 937 insertions(+), 637 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java new file mode 100644 index 0000000000000..c3151686c98bb --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.Settings; +import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; +import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RemoteClusterStateCleanupManagerIT extends RemoteStoreBaseIntegTestCase { + + private static final String INDEX_NAME = "test-index"; + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build(); + } + + private Map initialTestSetup(int shardCount, int replicaCount, int dataNodeCount, int clusterManagerNodeCount) { + prepareCluster(clusterManagerNodeCount, dataNodeCount, INDEX_NAME, replicaCount, shardCount); + Map indexStats = indexData(1, false, INDEX_NAME); + assertEquals(shardCount * (replicaCount + 1), getNumShards(INDEX_NAME).totalNumShards); + ensureGreen(INDEX_NAME); + return indexStats; + } + + public void testRemoteCleanupTaskUpdated() { + int shardCount = randomIntBetween(1, 2); + int replicaCount = 1; + int dataNodeCount = shardCount * (replicaCount + 1); + int clusterManagerNodeCount = 1; + + initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); + RemoteClusterStateCleanupManager remoteClusterStateCleanupManager = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateCleanupManager.class + ); + + assertEquals(CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval()); + assertTrue(remoteClusterStateCleanupManager.getStaleFileDeletionTask().isScheduled()); + + // now disable + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), -1)) + .get(); + + assertEquals(-1, remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval().getMillis()); + assertFalse(remoteClusterStateCleanupManager.getStaleFileDeletionTask().isScheduled()); + + // now set Clean up interval to 1 min + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) + .get(); + assertEquals(1, remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval().getMinutes()); + } + + public void testRemoteCleanupOnlyAfter10Updates() throws Exception { + int shardCount = randomIntBetween(1, 2); + int replicaCount = 1; + int dataNodeCount = shardCount * (replicaCount + 1); + int clusterManagerNodeCount = 1; + + initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); + + // set cleanup interval to 100 ms + ClusterUpdateSettingsResponse response = client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "100ms")) + .get(); + + assertEquals(true, response.isAcknowledged()); + + replicaCount = updateReplicaCountNTimes(9, replicaCount); + + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); + + BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); + BlobPath baseMetadataPath = repository.basePath() + .add( + Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) + ) + .add("cluster-state") + .add(getClusterState().metadata().clusterUUID()); + BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); + + assertBusy(() -> { + assertEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); + }, 500, TimeUnit.MILLISECONDS); + + replicaCount = updateReplicaCountNTimes(8, replicaCount); + + // wait for 1 min, to ensure that clean up task ran and didn't clean up stale files because it was less than 10 + Thread.sleep(100); + assertNotEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); + + // Do 2 more updates, now since the total successful state changes are more than 10, stale files will be cleaned up + replicaCount = updateReplicaCountNTimes(2, replicaCount); + + assertBusy(() -> { + assertEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); + }, 100, TimeUnit.MILLISECONDS); + + RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( + RemoteClusterStateService.class + ); + Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( + cluster().getClusterName(), + getClusterState().metadata().clusterUUID() + ).getMetadata().getIndices(); + assertEquals(replicaCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); + assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); + } + + + + private void setReplicaCount(int replicaCount) { + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, replicaCount)) + .get(); + ensureGreen(INDEX_NAME); + } + + private int updateReplicaCountNTimes(int n, int initialCount) { + int newReplicaCount = randomIntBetween(0, 3); + ; + for (int i = 0; i < n; i++) { + while (newReplicaCount == initialCount) { + newReplicaCount = randomIntBetween(0, 3); + } + setReplicaCount(newReplicaCount); + initialCount = newReplicaCount; + } + return newReplicaCount; + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index fd9a71d77fd1b..dc1810b79c2aa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -10,25 +10,16 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.DiscoveryStats; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; -import org.opensearch.repositories.RepositoriesService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchIntegTestCase; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; -import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase { @@ -40,124 +31,6 @@ protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build(); } - private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { - internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); - internalCluster().startDataOnlyNodes(numDataOnlyNodes); - for (String index : indices.split(",")) { - createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); - ensureYellowAndNoInitializingShards(index); - ensureGreen(index); - } - } - - private Map initialTestSetup(int shardCount, int replicaCount, int dataNodeCount, int clusterManagerNodeCount) { - prepareCluster(clusterManagerNodeCount, dataNodeCount, INDEX_NAME, replicaCount, shardCount); - Map indexStats = indexData(1, false, INDEX_NAME); - assertEquals(shardCount * (replicaCount + 1), getNumShards(INDEX_NAME).totalNumShards); - ensureGreen(INDEX_NAME); - return indexStats; - } - - public void testRemoteCleanupTaskUpdated() { - int shardCount = randomIntBetween(1, 2); - int replicaCount = 1; - int dataNodeCount = shardCount * (replicaCount + 1); - int clusterManagerNodeCount = 1; - - initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); - RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( - RemoteClusterStateService.class - ); - - assertEquals(5, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); - assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); - - // now disable - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), -1)) - .get(); - - assertEquals(-1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMillis()); - assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); - - // now set Clean up interval to 1 min - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) - .get(); - assertEquals(1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); - } - - public void testRemoteCleanupOnlyAfter10Updates() throws Exception { - int shardCount = randomIntBetween(1, 2); - int replicaCount = 1; - int dataNodeCount = shardCount * (replicaCount + 1); - int clusterManagerNodeCount = 1; - - initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); - RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( - RemoteClusterStateService.class - ); - - // set cleanup interval to 1 min - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) - .get(); - - replicaCount = updateReplicaCountNTimes(9, replicaCount); - - RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); - - BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); - BlobPath baseMetadataPath = repository.basePath() - .add( - Base64.getUrlEncoder() - .withoutPadding() - .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) - ) - .add("cluster-state") - .add(getClusterState().metadata().clusterUUID()); - BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); - - assertBusy(() -> { - assertEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); - }, 1, TimeUnit.MINUTES); - - replicaCount = updateReplicaCountNTimes(8, replicaCount); - - // wait for 1 min, to ensure that clean up task ran and didn't clean up stale files because it was less than 10 - Thread.sleep(60000); - assertNotEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); - - // Do 2 more updates, now since the total successful state changes are more than 10, stale files will be cleaned up - replicaCount = updateReplicaCountNTimes(2, replicaCount); - - assertBusy(() -> { - assertEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); - }, 1, TimeUnit.MINUTES); - - Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( - cluster().getClusterName(), - getClusterState().metadata().clusterUUID() - ).getMetadata().getIndices(); - assertEquals(replicaCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); - assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); - } - public void testRemoteStateStats() { int shardCount = randomIntBetween(1, 2); int replicaCount = 1; @@ -248,25 +121,4 @@ private void validateNodesStatsResponse(NodesStatsResponse nodesStatsResponse) { assertNotNull(nodesStatsResponse.getNodes().get(0)); assertNotNull(nodesStatsResponse.getNodes().get(0).getDiscoveryStats()); } - - private void setReplicaCount(int replicaCount) { - client().admin() - .indices() - .prepareUpdateSettings(INDEX_NAME) - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, replicaCount)) - .get(); - } - - private int updateReplicaCountNTimes(int n, int initialCount) { - int newReplicaCount = randomIntBetween(0, 3); - ; - for (int i = 0; i < n; i++) { - while (newReplicaCount == initialCount) { - newReplicaCount = randomIntBetween(0, 3); - } - setReplicaCount(newReplicaCount); - initialCount = newReplicaCount; - } - return newReplicaCount; - } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index d7ad0daa43524..501e762ea24e4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -311,4 +311,14 @@ protected void restore(boolean restoreAllShards, String... indices) { PlainActionFuture.newFuture() ); } + + protected void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { + internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); + internalCluster().startDataOnlyNodes(numDataOnlyNodes); + for (String index : indices.split(",")) { + createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); + ensureYellowAndNoInitializingShards(index); + ensureGreen(index); + } + } } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 2e82314106492..b2d407781843d 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -103,6 +103,7 @@ import org.opensearch.gateway.DanglingIndicesState; import org.opensearch.gateway.GatewayService; import org.opensearch.gateway.PersistedClusterStateService; +import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager; import org.opensearch.gateway.ShardsBatchGatewayAllocator; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.http.HttpTransportSettings; @@ -711,8 +712,8 @@ public void apply(Settings value, Settings current, Settings previous) { SearchRequestSlowLog.CLUSTER_SEARCH_REQUEST_SLOWLOG_LEVEL, // Remote cluster state settings + RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, - RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteClusterStateService.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java new file mode 100644 index 0000000000000..1e0c3dd325d0a --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -0,0 +1,365 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Strings; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.service.ClusterApplierService; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.AbstractAsyncTask; +import org.opensearch.core.action.ActionListener; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.threadpool.ThreadPool; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.opensearch.gateway.remote.RemoteClusterStateService.GLOBAL_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateService.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_PATH_TOKEN; + +/** + * A Manager which provides APIs to clean up stale cluster state files and runs an async stale cleanup task + * + * @opensearch.internal + */ +public class RemoteClusterStateCleanupManager implements Closeable { + + public static final int RETAINED_MANIFESTS = 10; + public static final int SKIP_CLEANUP_STATE_CHANGES = 10; + public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT = TimeValue.timeValueMillis(60000); + public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM = TimeValue.MINUS_ONE; + + /** + * Setting to specify the interval to do run stale file cleanup job + * Min value -1 indicates that the stale file cleanup job should be disabled + */ + public static final Setting REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING = Setting.timeSetting( + "cluster.remote_store.state.cleanup_interval", + CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, + CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + private static final Logger logger = LogManager.getLogger(RemoteClusterStateCleanupManager.class); + private RemoteClusterStateService remoteClusterStateService; + private RemotePersistenceStats remoteStateStats; + private BlobStoreTransferService blobStoreTransferService; + private volatile TimeValue staleFileCleanupInterval; + private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); + private AsyncStaleFileDeletion staleFileDeletionTask; + private long lastCleanupAttemptStateVersion; + private final ThreadPool threadpool; + private final ClusterApplierService clusterApplierService; + + public RemoteClusterStateCleanupManager(RemoteClusterStateService remoteClusterStateService, ClusterService clusterService) { + this.remoteClusterStateService = remoteClusterStateService; + this.remoteStateStats = remoteClusterStateService.getStats(); + ClusterSettings clusterSettings = clusterService.getClusterSettings(); + this.clusterApplierService = clusterService.getClusterApplierService(); + this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); + this.threadpool = remoteClusterStateService.getThreadpool(); + // initialize with 0, a cleanup will be done when this node is elected master node and version is incremented more than threshold + this.lastCleanupAttemptStateVersion = 0; + clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, this::updateCleanupInterval); + } + + void start() { + staleFileDeletionTask = new AsyncStaleFileDeletion(this); + } + + @Override + public void close() throws IOException { + if (staleFileDeletionTask != null) { + staleFileDeletionTask.close(); + } + } + + private BlobStoreTransferService getBlobStoreTransferService() { + if (blobStoreTransferService == null) { + blobStoreTransferService = new BlobStoreTransferService(remoteClusterStateService.getBlobStore(), threadpool); + } + return blobStoreTransferService; + } + + private void updateCleanupInterval(TimeValue updatedInterval) { + this.staleFileCleanupInterval = updatedInterval; + logger.info("updated remote state cleanup interval to {}", updatedInterval); + // After updating the interval, we need to close the current task and create a new one which will run with updated interval + if (staleFileDeletionTask != null && !staleFileDeletionTask.getInterval().equals(updatedInterval)) { + staleFileDeletionTask.setInterval(updatedInterval); + } + } + + void cleanUpStaleFiles() { + ClusterState currentAppliedState = clusterApplierService.state(); + if (currentAppliedState.nodes().isLocalNodeElectedClusterManager()) { + long cleanUpAttemptStateVersion = currentAppliedState.version(); + if ( + cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion > SKIP_CLEANUP_STATE_CHANGES && + Strings.isNotEmpty(currentAppliedState.getClusterName().value()) && + Strings.isNotEmpty(currentAppliedState.metadata().clusterUUID()) + ) { + logger.info( + "Cleaning up stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates", + currentAppliedState.getClusterName().value(), + currentAppliedState.metadata().clusterUUID(), + cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion + ); + deleteStaleClusterMetadata( + currentAppliedState.getClusterName().value(), + currentAppliedState.metadata().clusterUUID(), + RETAINED_MANIFESTS + ); + lastCleanupAttemptStateVersion = cleanUpAttemptStateVersion; + } else { + logger.debug( + "Skipping cleanup of stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates, which is less than threshold {}", + currentAppliedState.getClusterName().value(), + currentAppliedState.metadata().clusterUUID(), + cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion, + SKIP_CLEANUP_STATE_CHANGES + ); + } + } else { + logger.debug("Skipping cleanup task as local node is not elected Cluster Manager"); + } + } + + private void deleteClusterMetadata( + String clusterName, + String clusterUUID, + List activeManifestBlobMetadata, + List staleManifestBlobMetadata + ) { + try { + Set filesToKeep = new HashSet<>(); + Set staleManifestPaths = new HashSet<>(); + Set staleIndexMetadataPaths = new HashSet<>(); + Set staleGlobalMetadataPaths = new HashSet<>(); + activeManifestBlobMetadata.forEach(blobMetadata -> { + ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.fetchRemoteClusterMetadataManifest( + clusterName, + clusterUUID, + blobMetadata.name() + ); + clusterMetadataManifest.getIndices() + .forEach(uploadedIndexMetadata -> filesToKeep.add(uploadedIndexMetadata.getUploadedFilename())); + filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName()); + }); + staleManifestBlobMetadata.forEach(blobMetadata -> { + ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.fetchRemoteClusterMetadataManifest( + clusterName, + clusterUUID, + blobMetadata.name() + ); + staleManifestPaths.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blobMetadata.name()); + if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) { + String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); + staleGlobalMetadataPaths.add( + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( + globalMetadataSplitPath[globalMetadataSplitPath.length - 1] + ) + ); + } + clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { + if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) { + staleIndexMetadataPaths.add( + new BlobPath().add(INDEX_PATH_TOKEN).add(uploadedIndexMetadata.getIndexUUID()).buildAsString() + + INDEX_METADATA_FORMAT.blobName(uploadedIndexMetadata.getUploadedFilename()) + ); + } + }); + }); + + if (staleManifestPaths.isEmpty()) { + logger.debug("No stale Remote Cluster Metadata files found"); + return; + } + + deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleGlobalMetadataPaths)); + deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexMetadataPaths)); + deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleManifestPaths)); + } catch (IllegalStateException e) { + logger.error("Error while fetching Remote Cluster Metadata manifests", e); + } catch (IOException e) { + logger.error("Error while deleting stale Remote Cluster Metadata files", e); + remoteStateStats.cleanUpAttemptFailed(); + } catch (Exception e) { + logger.error("Unexpected error while deleting stale Remote Cluster Metadata files", e); + remoteStateStats.cleanUpAttemptFailed(); + } + } + + /** + * Deletes older than last {@code versionsToRetain} manifests. Also cleans up unreferenced IndexMetadata associated with older manifests + * + * @param clusterName name of the cluster + * @param clusterUUID uuid of cluster state to refer to in remote + * @param manifestsToRetain no of latest manifest files to keep in remote + */ + // package private for testing + void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int manifestsToRetain) { + if (deleteStaleMetadataRunning.compareAndSet(false, true) == false) { + logger.info("Delete stale cluster metadata task is already in progress."); + return; + } + try { + getBlobStoreTransferService().listAllInSortedOrderAsync( + ThreadPool.Names.REMOTE_PURGE, + remoteClusterStateService.getManifestFolderPath(clusterName, clusterUUID), + "manifest", + Integer.MAX_VALUE, + new ActionListener<>() { + @Override + public void onResponse(List blobMetadata) { + if (blobMetadata.size() > manifestsToRetain) { + deleteClusterMetadata( + clusterName, + clusterUUID, + blobMetadata.subList(0, manifestsToRetain - 1), + blobMetadata.subList(manifestsToRetain - 1, blobMetadata.size()) + ); + } + deleteStaleMetadataRunning.set(false); + } + + @Override + public void onFailure(Exception e) { + logger.error( + new ParameterizedMessage( + "Exception occurred while deleting Remote Cluster Metadata for clusterUUIDs {}", + clusterUUID + ) + ); + deleteStaleMetadataRunning.set(false); + } + } + ); + } catch (Exception e) { + deleteStaleMetadataRunning.set(false); + throw e; + } + } + + /** + * Purges all remote cluster state against provided cluster UUIDs + * + * @param clusterName name of the cluster + * @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged + */ + void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUIDs) { + clusterUUIDs.forEach(clusterUUID -> { + getBlobStoreTransferService().deleteAsync( + ThreadPool.Names.REMOTE_PURGE, + remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), + new ActionListener<>() { + @Override + public void onResponse(Void unused) { + logger.info("Deleted all remote cluster metadata for cluster UUID - {}", clusterUUID); + } + + @Override + public void onFailure(Exception e) { + logger.error( + new ParameterizedMessage( + "Exception occurred while deleting all remote cluster metadata for cluster UUID {}", + clusterUUID + ), + e + ); + remoteStateStats.cleanUpAttemptFailed(); + } + } + ); + }); + } + + + private void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { + logger.debug(String.format(Locale.ROOT, "Deleting stale files from remote - %s", stalePaths)); + getBlobStoreTransferService().deleteBlobs(remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), stalePaths); + } + + /** + * Purges all remote cluster state against provided cluster UUIDs + * @param clusterState current state of the cluster + * @param committedManifest last committed ClusterMetadataManifest + */ + public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataManifest committedManifest) { + threadpool.executor(ThreadPool.Names.REMOTE_PURGE).execute(() -> { + String clusterName = clusterState.getClusterName().value(); + logger.debug("Deleting stale cluster UUIDs data from remote [{}]", clusterName); + Set allClustersUUIDsInRemote; + try { + allClustersUUIDsInRemote = new HashSet<>(remoteClusterStateService.getAllClusterUUIDs(clusterState.getClusterName().value())); + } catch (IOException e) { + logger.info(String.format(Locale.ROOT, "Error while fetching all cluster UUIDs for [%s]", clusterName)); + return; + } + // Retain last 2 cluster uuids data + allClustersUUIDsInRemote.remove(committedManifest.getClusterUUID()); + allClustersUUIDsInRemote.remove(committedManifest.getPreviousClusterUUID()); + deleteStaleUUIDsClusterMetadata(clusterName, new ArrayList<>(allClustersUUIDsInRemote)); + }); + } + + + public TimeValue getStaleFileCleanupInterval() { + return this.staleFileCleanupInterval; + } + + AsyncStaleFileDeletion getStaleFileDeletionTask() { // for testing + return this.staleFileDeletionTask; + } + + RemotePersistenceStats getStats() { + return this.remoteStateStats; + } + + static final class AsyncStaleFileDeletion extends AbstractAsyncTask { + private final RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; + AsyncStaleFileDeletion(RemoteClusterStateCleanupManager remoteClusterStateCleanupManager) { + super( + logger, + remoteClusterStateCleanupManager.threadpool, + remoteClusterStateCleanupManager.getStaleFileCleanupInterval(), + true + ); + this.remoteClusterStateCleanupManager = remoteClusterStateCleanupManager; + rescheduleIfNecessary(); + } + + @Override + protected boolean mustReschedule() { + return true; + } + + @Override + protected void runInternal() { + remoteClusterStateCleanupManager.cleanUpStaleFiles(); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c0b63fe8f93a8..ed71df046d39c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -16,17 +16,17 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; @@ -49,7 +49,6 @@ import java.util.Base64; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -58,7 +57,6 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.LongSupplier; @@ -79,9 +77,6 @@ public class RemoteClusterStateService implements Closeable { public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int RETAINED_MANIFESTS = 10; - public static final int SKIP_CLEANUP_STATE_CHANGES = 10; - public static final String DELIMITER = "__"; private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); @@ -91,8 +86,6 @@ public class RemoteClusterStateService implements Closeable { public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); - public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT = TimeValue.timeValueMillis(60000); - public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM = TimeValue.MINUS_ONE; public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( "cluster.remote_store.state.index_metadata.upload_timeout", @@ -152,16 +145,6 @@ public class RemoteClusterStateService implements Closeable { Property.Final ); - /** - * Setting to specify the interval to do run stale file cleanup job - */ - public static final Setting REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING = Setting.timeSetting( - "cluster.remote_store.state.cleanup_interval", - CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, - CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM, - Property.NodeScope, - Property.Dynamic - ); public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; public static final String INDEX_PATH_TOKEN = "index"; @@ -184,18 +167,11 @@ public class RemoteClusterStateService implements Closeable { private volatile TimeValue indexMetadataUploadTimeout; private volatile TimeValue globalMetadataUploadTimeout; private volatile TimeValue metadataManifestUploadTimeout; - - private TimeValue staleFileCleanupInterval; - private AsyncStaleFileDeletion staleFileDeletionTask; - private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); + private RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; private final RemotePersistenceStats remoteStateStats; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; - private String latestClusterName; - private String latestClusterUUID; - private long lastCleanupAttemptState; - private boolean isClusterManagerNode; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. @@ -211,7 +187,7 @@ public RemoteClusterStateService( String nodeId, Supplier repositoriesService, Settings settings, - ClusterSettings clusterSettings, + ClusterService clusterService, LongSupplier relativeTimeNanosSupplier, ThreadPool threadPool, List indexMetadataUploadListeners @@ -222,6 +198,7 @@ public RemoteClusterStateService( this.settings = settings; this.relativeTimeNanosSupplier = relativeTimeNanosSupplier; this.threadpool = threadPool; + ClusterSettings clusterSettings = clusterService.getClusterSettings(); this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); @@ -231,19 +208,10 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); this.remoteStateStats = new RemotePersistenceStats(); - this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); - clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, this::updateCleanupInterval); - this.lastCleanupAttemptState = 0; - this.isClusterManagerNode = DiscoveryNode.isClusterManagerNode(settings); + this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService); this.indexMetadataUploadListeners = indexMetadataUploadListeners; } - private BlobStoreTransferService getBlobStoreTransferService() { - if (blobStoreTransferService == null) { - blobStoreTransferService = new BlobStoreTransferService(blobStoreRepository.blobStore(), threadpool); - } - return blobStoreTransferService; - } /** * This method uploads entire cluster state metadata to the configured blob store. For now only index metadata upload is supported. This method should be @@ -382,8 +350,6 @@ public ClusterMetadataManifest writeIncrementalMetadata( globalMetadataFile, false ); - this.latestClusterName = clusterState.getClusterName().value(); - this.latestClusterUUID = clusterState.metadata().clusterUUID(); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); @@ -646,46 +612,8 @@ private void writeIndexMetadataAsync( ); } - public TimeValue getStaleFileCleanupInterval() { - return this.staleFileCleanupInterval; - } - - AsyncStaleFileDeletion getStaleFileDeletionTask() { // for testing - return this.staleFileDeletionTask; - } - - private void updateCleanupInterval(TimeValue updatedInterval) { - if (!isClusterManagerNode) return; - this.staleFileCleanupInterval = updatedInterval; - logger.info("updated remote state cleanup interval to {}", updatedInterval); - // After updating the interval, we need to close the current task and create a new one which will run with updated interval - if (staleFileDeletionTask != null && !staleFileDeletionTask.getInterval().equals(updatedInterval)) { - staleFileDeletionTask.setInterval(updatedInterval); - } - } - - private void cleanUpStaleFiles() { - long cleanUpAttemptState = remoteStateStats.getSuccessCount(); - if (cleanUpAttemptState - lastCleanupAttemptState > SKIP_CLEANUP_STATE_CHANGES - && this.latestClusterName != null - && this.latestClusterUUID != null) { - logger.info( - "Cleaning up stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates", - this.latestClusterName, - this.latestClusterUUID, - cleanUpAttemptState - lastCleanupAttemptState - ); - deleteStaleClusterMetadata(this.latestClusterName, this.latestClusterUUID, RETAINED_MANIFESTS); - lastCleanupAttemptState = cleanUpAttemptState; - } else { - logger.info( - "Skipping cleanup of stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates, which is less than threshold {}", - this.latestClusterName, - this.latestClusterUUID, - cleanUpAttemptState - lastCleanupAttemptState, - SKIP_CLEANUP_STATE_CHANGES - ); - } + public RemoteClusterStateCleanupManager getCleanupManager() { + return remoteClusterStateCleanupManager; } @Nullable @@ -704,15 +632,13 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getGlobalMetadataFileName(), true ); - deleteStaleClusterUUIDs(clusterState, committedManifest); + remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifest); return committedManifest; } @Override public void close() throws IOException { - if (staleFileDeletionTask != null) { - staleFileDeletionTask.close(); - } + remoteClusterStateCleanupManager.close(); if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } @@ -727,9 +653,7 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; - if (isClusterManagerNode) { - staleFileDeletionTask = new AsyncStaleFileDeletion(this); - } + remoteClusterStateCleanupManager.start(); } private ClusterMetadataManifest uploadManifest( @@ -808,6 +732,24 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust ); } + ThreadPool getThreadpool() { + return threadpool; + } + + BlobStore getBlobStore() { + return blobStoreRepository.blobStore(); + } + + private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) { + final Optional latestManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID); + if (!latestManifest.isPresent()) { + final String previousClusterUUID = getLastKnownUUIDFromRemote(clusterName); + assert !clusterUUID.equals(previousClusterUUID) : "Last cluster UUID is same current cluster UUID"; + return previousClusterUUID; + } + return latestManifest.get().getPreviousClusterUUID(); + } + private BlobContainer indexMetadataContainer(String clusterName, String clusterUUID, String indexUUID) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX return blobStoreRepository.blobStore() @@ -825,7 +767,7 @@ private BlobContainer manifestContainer(String clusterName, String clusterUUID) return blobStoreRepository.blobStore().blobContainer(getManifestFolderPath(clusterName, clusterUUID)); } - private BlobPath getCusterMetadataBasePath(String clusterName, String clusterUUID) { + BlobPath getCusterMetadataBasePath(String clusterName, String clusterUUID) { return blobStoreRepository.basePath().add(encodeString(clusterName)).add(CLUSTER_STATE_PATH_TOKEN).add(clusterUUID); } @@ -904,7 +846,7 @@ private static String globalMetadataFileName(Metadata metadata) { ); } - private BlobPath getManifestFolderPath(String clusterName, String clusterUUID) { + BlobPath getManifestFolderPath(String clusterName, String clusterUUID) { return getCusterMetadataBasePath(clusterName, clusterUUID).add(MANIFEST_PATH_TOKEN); } @@ -1042,7 +984,7 @@ public String getLastKnownUUIDFromRemote(String clusterName) { } } - private Set getAllClusterUUIDs(String clusterName) throws IOException { + Set getAllClusterUUIDs(String clusterName) throws IOException { Map clusterUUIDMetadata = clusterUUIDContainer(clusterName).children(); if (clusterUUIDMetadata == null) { return Collections.emptySet(); @@ -1233,7 +1175,7 @@ private Optional getLatestManifestFileName(String clusterName, String cl * @param clusterName name of the cluster * @return ClusterMetadataManifest */ - private ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, String clusterUUID, String filename) + ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, String clusterUUID, String filename) throws IllegalStateException { try { return getClusterMetadataManifestBlobStoreFormat(filename).read( @@ -1291,205 +1233,7 @@ public RemoteStateTransferException(String errorDesc, Throwable cause) { } } - /** - * Purges all remote cluster state against provided cluster UUIDs - * - * @param clusterName name of the cluster - * @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged - */ - void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUIDs) { - clusterUUIDs.forEach(clusterUUID -> { - getBlobStoreTransferService().deleteAsync( - ThreadPool.Names.REMOTE_PURGE, - getCusterMetadataBasePath(clusterName, clusterUUID), - new ActionListener<>() { - @Override - public void onResponse(Void unused) { - logger.info("Deleted all remote cluster metadata for cluster UUID - {}", clusterUUID); - } - - @Override - public void onFailure(Exception e) { - logger.error( - new ParameterizedMessage( - "Exception occurred while deleting all remote cluster metadata for cluster UUID {}", - clusterUUID - ), - e - ); - remoteStateStats.cleanUpAttemptFailed(); - } - } - ); - }); - } - - /** - * Deletes older than last {@code versionsToRetain} manifests. Also cleans up unreferenced IndexMetadata associated with older manifests - * - * @param clusterName name of the cluster - * @param clusterUUID uuid of cluster state to refer to in remote - * @param manifestsToRetain no of latest manifest files to keep in remote - */ - // package private for testing - void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int manifestsToRetain) { - if (deleteStaleMetadataRunning.compareAndSet(false, true) == false) { - logger.info("Delete stale cluster metadata task is already in progress."); - return; - } - try { - getBlobStoreTransferService().listAllInSortedOrderAsync( - ThreadPool.Names.REMOTE_PURGE, - getManifestFolderPath(clusterName, clusterUUID), - "manifest", - Integer.MAX_VALUE, - new ActionListener<>() { - @Override - public void onResponse(List blobMetadata) { - if (blobMetadata.size() > manifestsToRetain) { - deleteClusterMetadata( - clusterName, - clusterUUID, - blobMetadata.subList(0, manifestsToRetain - 1), - blobMetadata.subList(manifestsToRetain - 1, blobMetadata.size()) - ); - } - deleteStaleMetadataRunning.set(false); - } - - @Override - public void onFailure(Exception e) { - logger.error( - new ParameterizedMessage( - "Exception occurred while deleting Remote Cluster Metadata for clusterUUIDs {}", - clusterUUID - ) - ); - deleteStaleMetadataRunning.set(false); - } - } - ); - } catch (Exception e) { - deleteStaleMetadataRunning.set(false); - throw e; - } - } - - private void deleteClusterMetadata( - String clusterName, - String clusterUUID, - List activeManifestBlobMetadata, - List staleManifestBlobMetadata - ) { - try { - Set filesToKeep = new HashSet<>(); - Set staleManifestPaths = new HashSet<>(); - Set staleIndexMetadataPaths = new HashSet<>(); - Set staleGlobalMetadataPaths = new HashSet<>(); - activeManifestBlobMetadata.forEach(blobMetadata -> { - ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest( - clusterName, - clusterUUID, - blobMetadata.name() - ); - clusterMetadataManifest.getIndices() - .forEach(uploadedIndexMetadata -> filesToKeep.add(uploadedIndexMetadata.getUploadedFilename())); - filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName()); - }); - staleManifestBlobMetadata.forEach(blobMetadata -> { - ClusterMetadataManifest clusterMetadataManifest = fetchRemoteClusterMetadataManifest( - clusterName, - clusterUUID, - blobMetadata.name() - ); - staleManifestPaths.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blobMetadata.name()); - if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) { - String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); - staleGlobalMetadataPaths.add( - new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( - globalMetadataSplitPath[globalMetadataSplitPath.length - 1] - ) - ); - } - clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { - if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) { - staleIndexMetadataPaths.add( - new BlobPath().add(INDEX_PATH_TOKEN).add(uploadedIndexMetadata.getIndexUUID()).buildAsString() - + INDEX_METADATA_FORMAT.blobName(uploadedIndexMetadata.getUploadedFilename()) - ); - } - }); - }); - - if (staleManifestPaths.isEmpty()) { - logger.debug("No stale Remote Cluster Metadata files found"); - return; - } - - deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleGlobalMetadataPaths)); - deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexMetadataPaths)); - deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleManifestPaths)); - } catch (IllegalStateException e) { - logger.error("Error while fetching Remote Cluster Metadata manifests", e); - } catch (IOException e) { - logger.error("Error while deleting stale Remote Cluster Metadata files", e); - remoteStateStats.cleanUpAttemptFailed(); - } catch (Exception e) { - logger.error("Unexpected error while deleting stale Remote Cluster Metadata files", e); - remoteStateStats.cleanUpAttemptFailed(); - } - } - - private void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { - logger.debug(String.format(Locale.ROOT, "Deleting stale files from remote - %s", stalePaths)); - getBlobStoreTransferService().deleteBlobs(getCusterMetadataBasePath(clusterName, clusterUUID), stalePaths); - } - - /** - * Purges all remote cluster state against provided cluster UUIDs - * - * @param clusterState current state of the cluster - * @param committedManifest last committed ClusterMetadataManifest - */ - public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataManifest committedManifest) { - threadpool.executor(ThreadPool.Names.REMOTE_PURGE).execute(() -> { - String clusterName = clusterState.getClusterName().value(); - logger.debug("Deleting stale cluster UUIDs data from remote [{}]", clusterName); - Set allClustersUUIDsInRemote; - try { - allClustersUUIDsInRemote = new HashSet<>(getAllClusterUUIDs(clusterState.getClusterName().value())); - } catch (IOException e) { - logger.info(String.format(Locale.ROOT, "Error while fetching all cluster UUIDs for [%s]", clusterName)); - return; - } - // Retain last 2 cluster uuids data - allClustersUUIDsInRemote.remove(committedManifest.getClusterUUID()); - allClustersUUIDsInRemote.remove(committedManifest.getPreviousClusterUUID()); - deleteStaleUUIDsClusterMetadata(clusterName, new ArrayList<>(allClustersUUIDsInRemote)); - }); - } - public RemotePersistenceStats getStats() { return remoteStateStats; } - - static final class AsyncStaleFileDeletion extends AbstractAsyncTask { - private final RemoteClusterStateService remoteClusterStateService; - - AsyncStaleFileDeletion(RemoteClusterStateService remoteClusterStateService) { - super(logger, remoteClusterStateService.threadpool, remoteClusterStateService.getStaleFileCleanupInterval(), true); - this.remoteClusterStateService = remoteClusterStateService; - rescheduleIfNecessary(); - } - - @Override - protected boolean mustReschedule() { - return true; - } - - @Override - protected void runInternal() { - remoteClusterStateService.cleanUpStaleFiles(); - } - } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 614f39166ea66..bc94dd570ae10 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -136,6 +136,7 @@ import org.opensearch.gateway.GatewayService; import org.opensearch.gateway.MetaStateService; import org.opensearch.gateway.PersistedClusterStateService; +import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager; import org.opensearch.gateway.ShardsBatchGatewayAllocator; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.http.HttpServerTransport; @@ -728,6 +729,7 @@ protected Node( threadPool::relativeTimeInMillis ); final RemoteClusterStateService remoteClusterStateService; + final RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; final RemoteIndexPathUploader remoteIndexPathUploader; if (isRemoteStoreClusterStateEnabled(settings)) { remoteIndexPathUploader = new RemoteIndexPathUploader( @@ -740,14 +742,16 @@ protected Node( nodeEnvironment.nodeId(), repositoriesServiceReference::get, settings, - clusterService.getClusterSettings(), + clusterService, threadPool::preciseRelativeTimeInNanos, threadPool, List.of(remoteIndexPathUploader) ); + remoteClusterStateCleanupManager = remoteClusterStateService.getCleanupManager(); } else { remoteClusterStateService = null; remoteIndexPathUploader = null; + remoteClusterStateCleanupManager = null; } // collect engine factory providers from plugins @@ -1326,6 +1330,7 @@ protected Node( b.bind(MetricsRegistry.class).toInstance(metricsRegistry); b.bind(RemoteClusterStateService.class).toProvider(() -> remoteClusterStateService); b.bind(RemoteIndexPathUploader.class).toProvider(() -> remoteIndexPathUploader); + b.bind(RemoteClusterStateCleanupManager.class).toProvider(() -> remoteClusterStateCleanupManager); b.bind(PersistedStateRegistry.class).toInstance(persistedStateRegistry); b.bind(SegmentReplicationStatsTracker.class).toInstance(segmentReplicationStatsTracker); b.bind(SearchRequestOperationsCompositeListenerFactory.class).toInstance(searchRequestOperationsCompositeListenerFactory); diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 3ba98c44f8d3e..ddde5a28b20ce 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -463,7 +463,7 @@ public void testDataOnlyNodePersistence() throws Exception { when(transportService.getThreadPool()).thenReturn(threadPool); ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()).thenReturn( - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) ); final PersistedClusterStateService persistedClusterStateService = new PersistedClusterStateService( nodeEnvironment, @@ -487,7 +487,7 @@ public void testDataOnlyNodePersistence() throws Exception { nodeEnvironment.nodeId(), repositoriesServiceSupplier, settings, - clusterSettings, + clusterService, () -> 0L, threadPool, List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java new file mode 100644 index 0000000000000..84cea32ad97e4 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -0,0 +1,335 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterApplierService; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.AbstractAsyncTask; +import org.opensearch.core.action.ActionListener; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.VersionUtils; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.rmi.Remote; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES; +import static org.opensearch.gateway.remote.RemoteClusterStateService.CLUSTER_STATE_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteClusterStateService.encodeString; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; +import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; + +public class RemoteClusterStateCleanupManagerTests extends OpenSearchTestCase { + private RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; + private Supplier repositoriesServiceSupplier; + private RepositoriesService repositoriesService; + private BlobStoreRepository blobStoreRepository; + private BlobStore blobStore; + private ClusterSettings clusterSettings; + private ClusterApplierService clusterApplierService; + private RemoteClusterStateService remoteClusterStateService; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + repositoriesServiceSupplier = mock(Supplier.class); + repositoriesService = mock(RepositoriesService.class); + when(repositoriesServiceSupplier.get()).thenReturn(repositoriesService); + + String stateRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + "remote_store_repository" + ); + String stateRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + "remote_store_repository" + ); + + Settings settings = Settings.builder() + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") + .put(stateRepoTypeAttributeKey, FsRepository.TYPE) + .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") + .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .build(); + + clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + clusterApplierService = mock(ClusterApplierService.class); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getClusterApplierService()).thenReturn(clusterApplierService); + + blobStoreRepository = mock(BlobStoreRepository.class); + blobStore = mock(BlobStore.class); + when(blobStoreRepository.blobStore()).thenReturn(blobStore); + when(repositoriesService.repository("remote_store_repository")).thenReturn(blobStoreRepository); + + remoteClusterStateService = mock(RemoteClusterStateService.class); + when(remoteClusterStateService.getStats()).thenReturn(new RemotePersistenceStats()); + when(remoteClusterStateService.getThreadpool()).thenReturn(threadPool); + when(remoteClusterStateService.getBlobStore()).thenReturn(blobStore); + remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(remoteClusterStateService, clusterService); + } + + @After + public void teardown() throws Exception { + super.tearDown(); + remoteClusterStateCleanupManager.close(); + threadPool.shutdown(); + } + + public void testDeleteStaleClusterUUIDs() throws IOException { + final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + ClusterMetadataManifest clusterMetadataManifest = ClusterMetadataManifest.builder() + .indices(List.of()) + .clusterTerm(1L) + .stateVersion(1L) + .stateUUID(randomAlphaOfLength(10)) + .clusterUUID("cluster-uuid1") + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID(ClusterState.UNKNOWN_UUID) + .committed(true) + .build(); + + BlobPath blobPath = new BlobPath().add("random-path"); + BlobContainer uuidContainerContainer = mock(BlobContainer.class); + BlobContainer manifest2Container = mock(BlobContainer.class); + BlobContainer manifest3Container = mock(BlobContainer.class); + when(blobStore.blobContainer(any())).then(invocation -> { + BlobPath blobPath1 = invocation.getArgument(0); + if (blobPath1.buildAsString().endsWith("cluster-state/")) { + return uuidContainerContainer; + } else if (blobPath1.buildAsString().contains("cluster-state/cluster-uuid2/")) { + return manifest2Container; + } else if (blobPath1.buildAsString().contains("cluster-state/cluster-uuid3/")) { + return manifest3Container; + } else { + throw new IllegalArgumentException("Unexpected blob path " + blobPath1); + } + }); + when( + manifest2Container.listBlobsByPrefixInSortedOrder( + MANIFEST_FILE_PREFIX + DELIMITER, + Integer.MAX_VALUE, + BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC + ) + ).thenReturn(List.of(new PlainBlobMetadata("mainfest2", 1L))); + when( + manifest3Container.listBlobsByPrefixInSortedOrder( + MANIFEST_FILE_PREFIX + DELIMITER, + Integer.MAX_VALUE, + BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC + ) + ).thenReturn(List.of(new PlainBlobMetadata("mainfest3", 1L))); + Set uuids = new HashSet<>(Arrays.asList("cluster-uuid1", "cluster-uuid2", "cluster-uuid3")); + when(remoteClusterStateService.getAllClusterUUIDs(any())).thenReturn(uuids); + when(remoteClusterStateService.getCusterMetadataBasePath(any(), any())).then( + invocationOnMock -> + blobPath.add(encodeString(invocationOnMock.getArgument(0))) + .add(CLUSTER_STATE_PATH_TOKEN) + .add(invocationOnMock.getArgument(1)) + ); + remoteClusterStateCleanupManager.start(); + remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, clusterMetadataManifest); + try { + assertBusy(() -> { + verify(manifest2Container, times(1)).delete(); + verify(manifest3Container, times(1)).delete(); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + public void testRemoteStateCleanupFailureStats() throws IOException { + BlobContainer blobContainer = mock(BlobContainer.class); + doThrow(IOException.class).when(blobContainer).delete(); + when(blobStore.blobContainer(any())).thenReturn(blobContainer); + BlobPath blobPath = new BlobPath().add("random-path"); + when((blobStoreRepository.basePath())).thenReturn(blobPath); + remoteClusterStateCleanupManager.start(); + remoteClusterStateCleanupManager.deleteStaleUUIDsClusterMetadata("cluster1", Arrays.asList("cluster-uuid1")); + try { + assertBusy(() -> { + // wait for stats to get updated + assertTrue(remoteClusterStateCleanupManager.getStats() != null); + assertEquals(0, remoteClusterStateCleanupManager.getStats().getSuccessCount()); + assertEquals(1, remoteClusterStateCleanupManager.getStats().getCleanupAttemptFailedCount()); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Exception { + BlobContainer blobContainer = mock(BlobContainer.class); + when(blobStore.blobContainer(any())).thenReturn(blobContainer); + + CountDownLatch latch = new CountDownLatch(1); + AtomicInteger callCount = new AtomicInteger(0); + doAnswer(invocation -> { + callCount.incrementAndGet(); + if (latch.await(5000, TimeUnit.SECONDS) == false) { + throw new Exception("Timed out waiting for delete task queuing to complete"); + } + return null; + }).when(blobContainer) + .listBlobsByPrefixInSortedOrder( + any(String.class), + any(int.class), + any(BlobContainer.BlobNameSortOrder.class), + any(ActionListener.class) + ); + + remoteClusterStateCleanupManager.start(); + remoteClusterStateCleanupManager.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS); + remoteClusterStateCleanupManager.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS); + + latch.countDown(); + assertBusy(() -> assertEquals(1, callCount.get())); + } + + + public void testRemoteClusterStateCleanupSetting() { + remoteClusterStateCleanupManager.start(); + // verify default value + assertEquals( + CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, + remoteClusterStateCleanupManager.getStaleFileCleanupInterval() + ); + + // verify update interval + int cleanupInterval = randomIntBetween(1, 10); + Settings newSettings = Settings.builder() + .put("cluster.remote_store.state.cleanup_interval", cleanupInterval + "s") + .build(); + clusterSettings.applySettings(newSettings); + assertEquals(cleanupInterval, remoteClusterStateCleanupManager.getStaleFileCleanupInterval().seconds()); + } + + public void testRemoteCleanupTaskScheduled() { + AbstractAsyncTask cleanupTask = remoteClusterStateCleanupManager.getStaleFileDeletionTask(); + assertNull(cleanupTask); + // now the task should be initialized + remoteClusterStateCleanupManager.start(); + assertNotNull(remoteClusterStateCleanupManager.getStaleFileDeletionTask()); + assertTrue(remoteClusterStateCleanupManager.getStaleFileDeletionTask().mustReschedule()); + assertEquals( + clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING), + remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval() + ); + assertTrue(remoteClusterStateCleanupManager.getStaleFileDeletionTask().isScheduled()); + assertFalse(remoteClusterStateCleanupManager.getStaleFileDeletionTask().isClosed()); + } + + public void testRemoteCleanupSkipsOnNonElectedClusterManager() { + ClusterState clusterState = mock(ClusterState.class); + when(clusterApplierService.state()).thenReturn(clusterState); + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + when(nodes.isLocalNodeElectedClusterManager()).thenReturn(false); + when(clusterState.nodes()).thenReturn(nodes); + RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); + AtomicInteger callCount = new AtomicInteger(0); + doAnswer(invocation -> callCount.incrementAndGet()).when(spyManager).deleteStaleClusterMetadata(any(), any(), anyInt()); + remoteClusterStateCleanupManager.cleanUpStaleFiles(); + assertEquals(0, callCount.get()); + } + + public void testRemoteCleanupSkipsIfVersionIncrementLessThanThreshold() { + ClusterState clusterState = mock(ClusterState.class); + Metadata metadata = mock(Metadata.class); + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + long version = randomLongBetween(1, SKIP_CLEANUP_STATE_CHANGES); + when(clusterApplierService.state()).thenReturn(clusterState); + when(nodes.isLocalNodeElectedClusterManager()).thenReturn(true); + when(clusterState.nodes()).thenReturn(nodes); + when(clusterState.getClusterName()).thenReturn(ClusterName.DEFAULT); + when(metadata.clusterUUID()).thenReturn("test-uuid"); + when(clusterState.metadata()).thenReturn(metadata); + when(metadata.version()).thenReturn(version); + + RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); + AtomicInteger callCount = new AtomicInteger(0); + doAnswer(invocation -> callCount.incrementAndGet()).when(spyManager).deleteStaleClusterMetadata(any(), any(), anyInt()); + + remoteClusterStateCleanupManager.cleanUpStaleFiles(); + assertEquals(0, callCount.get()); + } + + public void testRemoteCleanupCallsDeleteIfVersionIncrementGreaterThanThreshold() { + ClusterState clusterState = mock(ClusterState.class); + Metadata metadata = mock(Metadata.class); + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + long version = randomLongBetween(SKIP_CLEANUP_STATE_CHANGES + 1, SKIP_CLEANUP_STATE_CHANGES + 10); + when(clusterApplierService.state()).thenReturn(clusterState); + when(nodes.isLocalNodeElectedClusterManager()).thenReturn(true); + when(clusterState.nodes()).thenReturn(nodes); + when(clusterState.getClusterName()).thenReturn(ClusterName.DEFAULT); + when(metadata.clusterUUID()).thenReturn("test-uuid"); + when(clusterState.metadata()).thenReturn(metadata); + when(metadata.version()).thenReturn(version); + + RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); + AtomicInteger callCount = new AtomicInteger(0); + doAnswer(invocation -> callCount.incrementAndGet()).when(spyManager).deleteStaleClusterMetadata(any(), any(), anyInt()); + + // using spied cleanup manager so that stubbed deleteStaleClusterMetadata is called + spyManager.cleanUpStaleFiles(); + assertEquals(1, callCount.get()); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index bf03b99eb2109..faf266b63c67a 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; @@ -31,7 +32,6 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -41,7 +41,6 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; @@ -68,9 +67,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -80,17 +76,12 @@ import org.mockito.ArgumentMatchers; import static java.util.stream.Collectors.toList; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.opensearch.gateway.remote.RemoteClusterStateService.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; -import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; -import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -103,13 +94,12 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class RemoteClusterStateServiceTests extends OpenSearchTestCase { private RemoteClusterStateService remoteClusterStateService; + private ClusterService clusterService; private ClusterSettings clusterSettings; private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; @@ -142,6 +132,8 @@ public void setup() { .build(); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); NamedXContentRegistry xContentRegistry = new NamedXContentRegistry( Stream.of( NetworkModule.getNamedXContents().stream(), @@ -159,7 +151,7 @@ public void setup() { "test-node-id", repositoriesServiceSupplier, settings, - clusterSettings, + clusterService, () -> 0L, threadPool, List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) @@ -181,14 +173,15 @@ public void testFailWriteFullMetadataNonClusterManagerNode() throws IOException public void testFailInitializationWhenRemoteStateDisabled() { final Settings settings = Settings.builder().build(); - ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(clusterService.getClusterSettings()) + .thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); assertThrows( AssertionError.class, () -> new RemoteClusterStateService( "test-node-id", repositoriesServiceSupplier, settings, - clusterSettings, + clusterService, () -> 0L, threadPool, List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) @@ -1018,72 +1011,6 @@ public void testGetValidPreviousClusterUUIDWhenLastUUIDUncommitted() throws IOEx assertThat(previousClusterUUID, equalTo("cluster-uuid2")); } - public void testDeleteStaleClusterUUIDs() throws IOException { - final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); - ClusterMetadataManifest clusterMetadataManifest = ClusterMetadataManifest.builder() - .indices(List.of()) - .clusterTerm(1L) - .stateVersion(1L) - .stateUUID(randomAlphaOfLength(10)) - .clusterUUID("cluster-uuid1") - .nodeId("nodeA") - .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) - .previousClusterUUID(ClusterState.UNKNOWN_UUID) - .committed(true) - .build(); - - BlobPath blobPath = new BlobPath().add("random-path"); - when((blobStoreRepository.basePath())).thenReturn(blobPath); - BlobContainer uuidContainerContainer = mock(BlobContainer.class); - BlobContainer manifest2Container = mock(BlobContainer.class); - BlobContainer manifest3Container = mock(BlobContainer.class); - when(blobStore.blobContainer(any())).then(invocation -> { - BlobPath blobPath1 = invocation.getArgument(0); - if (blobPath1.buildAsString().endsWith("cluster-state/")) { - return uuidContainerContainer; - } else if (blobPath1.buildAsString().contains("cluster-state/cluster-uuid2/")) { - return manifest2Container; - } else if (blobPath1.buildAsString().contains("cluster-state/cluster-uuid3/")) { - return manifest3Container; - } else { - throw new IllegalArgumentException("Unexpected blob path " + blobPath1); - } - }); - Map blobMetadataMap = Map.of( - "cluster-uuid1", - mock(BlobContainer.class), - "cluster-uuid2", - mock(BlobContainer.class), - "cluster-uuid3", - mock(BlobContainer.class) - ); - when(uuidContainerContainer.children()).thenReturn(blobMetadataMap); - when( - manifest2Container.listBlobsByPrefixInSortedOrder( - MANIFEST_FILE_PREFIX + DELIMITER, - Integer.MAX_VALUE, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ) - ).thenReturn(List.of(new PlainBlobMetadata("mainfest2", 1L))); - when( - manifest3Container.listBlobsByPrefixInSortedOrder( - MANIFEST_FILE_PREFIX + DELIMITER, - Integer.MAX_VALUE, - BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC - ) - ).thenReturn(List.of(new PlainBlobMetadata("mainfest3", 1L))); - remoteClusterStateService.start(); - remoteClusterStateService.deleteStaleClusterUUIDs(clusterState, clusterMetadataManifest); - try { - assertBusy(() -> { - verify(manifest2Container, times(1)).delete(); - verify(manifest3Container, times(1)).delete(); - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - public void testRemoteStateStats() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); @@ -1096,26 +1023,6 @@ public void testRemoteStateStats() throws IOException { assertEquals(0, remoteClusterStateService.getStats().getFailedCount()); } - public void testRemoteStateCleanupFailureStats() throws IOException { - BlobContainer blobContainer = mock(BlobContainer.class); - doThrow(IOException.class).when(blobContainer).delete(); - when(blobStore.blobContainer(any())).thenReturn(blobContainer); - BlobPath blobPath = new BlobPath().add("random-path"); - when((blobStoreRepository.basePath())).thenReturn(blobPath); - remoteClusterStateService.start(); - remoteClusterStateService.deleteStaleUUIDsClusterMetadata("cluster1", Arrays.asList("cluster-uuid1")); - try { - assertBusy(() -> { - // wait for stats to get updated - assertTrue(remoteClusterStateService.getStats() != null); - assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); - assertEquals(1, remoteClusterStateService.getStats().getCleanupAttemptFailedCount()); - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - public void testFileNames() { final Index index = new Index("test-index", "index-uuid"); final Settings idxSettings = Settings.builder() @@ -1150,36 +1057,6 @@ public void testFileNames() { assertThat(splittedName[3], is("P")); } - public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Exception { - BlobContainer blobContainer = mock(BlobContainer.class); - BlobPath blobPath = new BlobPath().add("random-path"); - when((blobStoreRepository.basePath())).thenReturn(blobPath); - when(blobStore.blobContainer(any())).thenReturn(blobContainer); - - CountDownLatch latch = new CountDownLatch(1); - AtomicInteger callCount = new AtomicInteger(0); - doAnswer(invocation -> { - callCount.incrementAndGet(); - if (latch.await(5000, TimeUnit.SECONDS) == false) { - throw new Exception("Timed out waiting for delete task queuing to complete"); - } - return null; - }).when(blobContainer) - .listBlobsByPrefixInSortedOrder( - any(String.class), - any(int.class), - any(BlobContainer.BlobNameSortOrder.class), - any(ActionListener.class) - ); - - remoteClusterStateService.start(); - remoteClusterStateService.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS); - remoteClusterStateService.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS); - - latch.countDown(); - assertBusy(() -> assertEquals(1, callCount.get())); - } - public void testIndexMetadataUploadWaitTimeSetting() { // verify default value assertEquals( @@ -1228,69 +1105,6 @@ public void testGlobalMetadataUploadWaitTimeSetting() { assertEquals(globalMetadataUploadTimeout, remoteClusterStateService.getGlobalMetadataUploadTimeout().seconds()); } - public void testRemoteClusterStateCleanupSetting() { - remoteClusterStateService.start(); - // verify default value - assertEquals( - CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, - remoteClusterStateService.getStaleFileCleanupInterval() - ); - - // verify update interval - int cleanupInterval = randomIntBetween(1, 10); - Settings newSettings = Settings.builder() - .put("cluster.remote_store.state.cleanup_interval", cleanupInterval + "s") - .build(); - clusterSettings.applySettings(newSettings); - assertEquals(cleanupInterval, remoteClusterStateService.getStaleFileCleanupInterval().seconds()); - } - - public void testRemoteCleanupTaskScheduled() { - AbstractAsyncTask cleanupTask = remoteClusterStateService.getStaleFileDeletionTask(); - assertNull(cleanupTask); - // now the task should be initialized - remoteClusterStateService.start(); - assertNotNull(remoteClusterStateService.getStaleFileDeletionTask()); - assertTrue(remoteClusterStateService.getStaleFileDeletionTask().mustReschedule()); - assertEquals( - clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING), - remoteClusterStateService.getStaleFileDeletionTask().getInterval() - ); - assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); - assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isClosed()); - } - - public void testRemoteCleanupNotInitializedOnDataOnlyNode() { - String stateRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - "remote_store_repository" - ); - String stateRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - "remote_store_repository" - ); - Settings settings = Settings.builder() - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") - .putList(NODE_ROLES_SETTING.getKey(), "d") - .put(stateRepoTypeAttributeKey, FsRepository.TYPE) - .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") - .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .build(); - ClusterSettings dataNodeClusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - remoteClusterStateService = new RemoteClusterStateService( - "test-node-id", - repositoriesServiceSupplier, - settings, - dataNodeClusterSettings, - () -> 0L, - threadPool - ); - remoteClusterStateService.start(); - assertNull(remoteClusterStateService.getStaleFileDeletionTask()); - } - private void mockObjectsForGettingPreviousClusterUUID(Map clusterUUIDsPointers) throws IOException { mockObjectsForGettingPreviousClusterUUID(clusterUUIDsPointers, false, Collections.emptyMap()); } @@ -1581,7 +1395,7 @@ private static ClusterState.Builder generateClusterStateWithGlobalMetadata() { ); } - private static ClusterState.Builder generateClusterStateWithOneIndex() { + static ClusterState.Builder generateClusterStateWithOneIndex() { final Index index = new Index("test-index", "index-uuid"); final Settings idxSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -1606,7 +1420,7 @@ private static ClusterState.Builder generateClusterStateWithOneIndex() { ); } - private static DiscoveryNodes nodesWithLocalNodeClusterManager() { + static DiscoveryNodes nodesWithLocalNodeClusterManager() { return DiscoveryNodes.builder().clusterManagerNodeId("cluster-manager-id").localNodeId("cluster-manager-id").build(); } From bb4b71f04d19c02e55691e7a038c972386ea4a66 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 29 Apr 2024 11:56:42 +0530 Subject: [PATCH 025/133] Add UT Signed-off-by: Shivansh Arora --- .../RemoteClusterStateCleanupManagerIT.java | 2 - .../remote/RemoteClusterStateServiceIT.java | 2 - .../common/settings/ClusterSettings.java | 2 +- .../RemoteClusterStateCleanupManager.java | 26 +++-- .../remote/RemoteClusterStateService.java | 2 - .../main/java/org/opensearch/node/Node.java | 2 +- .../GatewayMetaStatePersistedStateTests.java | 4 +- ...RemoteClusterStateCleanupManagerTests.java | 100 ++++++++++-------- .../RemoteClusterStateServiceTests.java | 3 +- 9 files changed, 74 insertions(+), 69 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java index c3151686c98bb..033aa0e186539 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -148,8 +148,6 @@ public void testRemoteCleanupOnlyAfter10Updates() throws Exception { assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); } - - private void setReplicaCount(int replicaCount) { client().admin() .indices() diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index dc1810b79c2aa..3a023670c4abb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -15,10 +15,8 @@ import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase; -import java.util.Map; import java.util.stream.Collectors; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index b2d407781843d..0e7580c3af098 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -103,8 +103,8 @@ import org.opensearch.gateway.DanglingIndicesState; import org.opensearch.gateway.GatewayService; import org.opensearch.gateway.PersistedClusterStateService; -import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager; import org.opensearch.gateway.ShardsBatchGatewayAllocator; +import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.http.HttpTransportSettings; import org.opensearch.index.IndexModule; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 1e0c3dd325d0a..0fc6c01231331 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -38,6 +38,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_PATH_TOKEN; /** @@ -113,22 +114,21 @@ private void updateCleanupInterval(TimeValue updatedInterval) { } } + // visible for testing void cleanUpStaleFiles() { ClusterState currentAppliedState = clusterApplierService.state(); if (currentAppliedState.nodes().isLocalNodeElectedClusterManager()) { long cleanUpAttemptStateVersion = currentAppliedState.version(); - if ( - cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion > SKIP_CLEANUP_STATE_CHANGES && - Strings.isNotEmpty(currentAppliedState.getClusterName().value()) && - Strings.isNotEmpty(currentAppliedState.metadata().clusterUUID()) - ) { + if (cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion > SKIP_CLEANUP_STATE_CHANGES + && Strings.isNotEmpty(currentAppliedState.getClusterName().value()) + && Strings.isNotEmpty(currentAppliedState.metadata().clusterUUID())) { logger.info( "Cleaning up stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates", currentAppliedState.getClusterName().value(), currentAppliedState.metadata().clusterUUID(), cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion ); - deleteStaleClusterMetadata( + this.deleteStaleClusterMetadata( currentAppliedState.getClusterName().value(), currentAppliedState.metadata().clusterUUID(), RETAINED_MANIFESTS @@ -230,7 +230,7 @@ void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int mani getBlobStoreTransferService().listAllInSortedOrderAsync( ThreadPool.Names.REMOTE_PURGE, remoteClusterStateService.getManifestFolderPath(clusterName, clusterUUID), - "manifest", + MANIFEST_FILE_PREFIX, Integer.MAX_VALUE, new ActionListener<>() { @Override @@ -297,10 +297,12 @@ public void onFailure(Exception e) { }); } - private void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { logger.debug(String.format(Locale.ROOT, "Deleting stale files from remote - %s", stalePaths)); - getBlobStoreTransferService().deleteBlobs(remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), stalePaths); + getBlobStoreTransferService().deleteBlobs( + remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), + stalePaths + ); } /** @@ -314,7 +316,9 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa logger.debug("Deleting stale cluster UUIDs data from remote [{}]", clusterName); Set allClustersUUIDsInRemote; try { - allClustersUUIDsInRemote = new HashSet<>(remoteClusterStateService.getAllClusterUUIDs(clusterState.getClusterName().value())); + allClustersUUIDsInRemote = new HashSet<>( + remoteClusterStateService.getAllClusterUUIDs(clusterState.getClusterName().value()) + ); } catch (IOException e) { logger.info(String.format(Locale.ROOT, "Error while fetching all cluster UUIDs for [%s]", clusterName)); return; @@ -326,7 +330,6 @@ public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataMa }); } - public TimeValue getStaleFileCleanupInterval() { return this.staleFileCleanupInterval; } @@ -341,6 +344,7 @@ RemotePersistenceStats getStats() { static final class AsyncStaleFileDeletion extends AbstractAsyncTask { private final RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; + AsyncStaleFileDeletion(RemoteClusterStateCleanupManager remoteClusterStateCleanupManager) { super( logger, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ed71df046d39c..d67a840b5a611 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -145,7 +145,6 @@ public class RemoteClusterStateService implements Closeable { Property.Final ); - public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; public static final String INDEX_PATH_TOKEN = "index"; public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; @@ -212,7 +211,6 @@ public RemoteClusterStateService( this.indexMetadataUploadListeners = indexMetadataUploadListeners; } - /** * This method uploads entire cluster state metadata to the configured blob store. For now only index metadata upload is supported. This method should be * invoked by the elected cluster manager when the remote cluster state is enabled. diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index bc94dd570ae10..44b9934fb9133 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -136,8 +136,8 @@ import org.opensearch.gateway.GatewayService; import org.opensearch.gateway.MetaStateService; import org.opensearch.gateway.PersistedClusterStateService; -import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager; import org.opensearch.gateway.ShardsBatchGatewayAllocator; +import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.http.HttpServerTransport; import org.opensearch.identity.IdentityService; diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index ddde5a28b20ce..418e6d8de6adb 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -462,9 +462,7 @@ public void testDataOnlyNodePersistence() throws Exception { }); when(transportService.getThreadPool()).thenReturn(threadPool); ClusterService clusterService = mock(ClusterService.class); - when(clusterService.getClusterSettings()).thenReturn( - new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ); + when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); final PersistedClusterStateService persistedClusterStateService = new PersistedClusterStateService( nodeEnvironment, xContentRegistry(), diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 84cea32ad97e4..b3e04abb6ad29 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -8,8 +8,6 @@ package org.opensearch.gateway.remote; -import org.junit.After; -import org.junit.Before; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; @@ -31,30 +29,21 @@ import org.opensearch.test.VersionUtils; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; import java.io.IOException; -import java.rmi.Remote; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.AsyncStaleFileDeletion; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; @@ -65,10 +54,18 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.encodeString; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; -import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class RemoteClusterStateCleanupManagerTests extends OpenSearchTestCase { private RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; @@ -78,6 +75,8 @@ public class RemoteClusterStateCleanupManagerTests extends OpenSearchTestCase { private BlobStore blobStore; private ClusterSettings clusterSettings; private ClusterApplierService clusterApplierService; + private ClusterState clusterState; + private Metadata metadata; private RemoteClusterStateService remoteClusterStateService; private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @@ -107,8 +106,14 @@ public void setup() { clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); clusterApplierService = mock(ClusterApplierService.class); + clusterState = mock(ClusterState.class); + metadata = mock(Metadata.class); ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterState.getClusterName()).thenReturn(new ClusterName("test")); + when(metadata.clusterUUID()).thenReturn("testUUID"); + when(clusterState.metadata()).thenReturn(metadata); + when(clusterApplierService.state()).thenReturn(clusterState); when(clusterService.getClusterApplierService()).thenReturn(clusterApplierService); blobStoreRepository = mock(BlobStoreRepository.class); @@ -177,10 +182,9 @@ public void testDeleteStaleClusterUUIDs() throws IOException { Set uuids = new HashSet<>(Arrays.asList("cluster-uuid1", "cluster-uuid2", "cluster-uuid3")); when(remoteClusterStateService.getAllClusterUUIDs(any())).thenReturn(uuids); when(remoteClusterStateService.getCusterMetadataBasePath(any(), any())).then( - invocationOnMock -> - blobPath.add(encodeString(invocationOnMock.getArgument(0))) - .add(CLUSTER_STATE_PATH_TOKEN) - .add(invocationOnMock.getArgument(1)) + invocationOnMock -> blobPath.add(encodeString(invocationOnMock.getArgument(0))) + .add(CLUSTER_STATE_PATH_TOKEN) + .add((String) invocationOnMock.getArgument(1)) ); remoteClusterStateCleanupManager.start(); remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, clusterMetadataManifest); @@ -194,7 +198,6 @@ public void testDeleteStaleClusterUUIDs() throws IOException { } } - public void testRemoteStateCleanupFailureStats() throws IOException { BlobContainer blobContainer = mock(BlobContainer.class); doThrow(IOException.class).when(blobContainer).delete(); @@ -215,7 +218,6 @@ public void testRemoteStateCleanupFailureStats() throws IOException { } } - public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Exception { BlobContainer blobContainer = mock(BlobContainer.class); when(blobStore.blobContainer(any())).thenReturn(blobContainer); @@ -244,20 +246,14 @@ public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Excepti assertBusy(() -> assertEquals(1, callCount.get())); } - public void testRemoteClusterStateCleanupSetting() { remoteClusterStateCleanupManager.start(); // verify default value - assertEquals( - CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, - remoteClusterStateCleanupManager.getStaleFileCleanupInterval() - ); + assertEquals(CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT, remoteClusterStateCleanupManager.getStaleFileCleanupInterval()); // verify update interval int cleanupInterval = randomIntBetween(1, 10); - Settings newSettings = Settings.builder() - .put("cluster.remote_store.state.cleanup_interval", cleanupInterval + "s") - .build(); + Settings newSettings = Settings.builder().put("cluster.remote_store.state.cleanup_interval", cleanupInterval + "s").build(); clusterSettings.applySettings(newSettings); assertEquals(cleanupInterval, remoteClusterStateCleanupManager.getStaleFileCleanupInterval().seconds()); } @@ -277,31 +273,29 @@ public void testRemoteCleanupTaskScheduled() { assertFalse(remoteClusterStateCleanupManager.getStaleFileDeletionTask().isClosed()); } - public void testRemoteCleanupSkipsOnNonElectedClusterManager() { - ClusterState clusterState = mock(ClusterState.class); - when(clusterApplierService.state()).thenReturn(clusterState); + public void testRemoteCleanupSkipsOnOnlyElectedClusterManager() { DiscoveryNodes nodes = mock(DiscoveryNodes.class); when(nodes.isLocalNodeElectedClusterManager()).thenReturn(false); when(clusterState.nodes()).thenReturn(nodes); RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); AtomicInteger callCount = new AtomicInteger(0); doAnswer(invocation -> callCount.incrementAndGet()).when(spyManager).deleteStaleClusterMetadata(any(), any(), anyInt()); - remoteClusterStateCleanupManager.cleanUpStaleFiles(); + spyManager.cleanUpStaleFiles(); assertEquals(0, callCount.get()); + + when(nodes.isLocalNodeElectedClusterManager()).thenReturn(true); + when(clusterState.version()).thenReturn(randomLongBetween(11, 20)); + spyManager.cleanUpStaleFiles(); + assertEquals(1, callCount.get()); } public void testRemoteCleanupSkipsIfVersionIncrementLessThanThreshold() { - ClusterState clusterState = mock(ClusterState.class); - Metadata metadata = mock(Metadata.class); DiscoveryNodes nodes = mock(DiscoveryNodes.class); long version = randomLongBetween(1, SKIP_CLEANUP_STATE_CHANGES); when(clusterApplierService.state()).thenReturn(clusterState); when(nodes.isLocalNodeElectedClusterManager()).thenReturn(true); when(clusterState.nodes()).thenReturn(nodes); - when(clusterState.getClusterName()).thenReturn(ClusterName.DEFAULT); - when(metadata.clusterUUID()).thenReturn("test-uuid"); - when(clusterState.metadata()).thenReturn(metadata); - when(metadata.version()).thenReturn(version); + when(clusterState.version()).thenReturn(version); RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); AtomicInteger callCount = new AtomicInteger(0); @@ -312,17 +306,12 @@ public void testRemoteCleanupSkipsIfVersionIncrementLessThanThreshold() { } public void testRemoteCleanupCallsDeleteIfVersionIncrementGreaterThanThreshold() { - ClusterState clusterState = mock(ClusterState.class); - Metadata metadata = mock(Metadata.class); DiscoveryNodes nodes = mock(DiscoveryNodes.class); long version = randomLongBetween(SKIP_CLEANUP_STATE_CHANGES + 1, SKIP_CLEANUP_STATE_CHANGES + 10); when(clusterApplierService.state()).thenReturn(clusterState); when(nodes.isLocalNodeElectedClusterManager()).thenReturn(true); when(clusterState.nodes()).thenReturn(nodes); - when(clusterState.getClusterName()).thenReturn(ClusterName.DEFAULT); - when(metadata.clusterUUID()).thenReturn("test-uuid"); - when(clusterState.metadata()).thenReturn(metadata); - when(metadata.version()).thenReturn(version); + when(clusterState.version()).thenReturn(version); RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); AtomicInteger callCount = new AtomicInteger(0); @@ -332,4 +321,25 @@ public void testRemoteCleanupCallsDeleteIfVersionIncrementGreaterThanThreshold() spyManager.cleanUpStaleFiles(); assertEquals(1, callCount.get()); } + + public void testRemoteCleanupSchedulesEvenAfterFailure() throws InterruptedException { + remoteClusterStateCleanupManager.start(); + RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); + AtomicInteger callCount = new AtomicInteger(0); + doAnswer(invocationOnMock -> { + callCount.incrementAndGet(); + throw new RuntimeException("Test exception"); + }).when(spyManager).cleanUpStaleFiles(); + AsyncStaleFileDeletion task = new AsyncStaleFileDeletion(spyManager); + assertTrue(task.isScheduled()); + task.run(); + // Task is still scheduled after the failure + assertTrue(task.isScheduled()); + assertEquals(1, callCount.get()); + + task.run(); + // Task is still scheduled after the failure + assertTrue(task.isScheduled()); + assertEquals(2, callCount.get()); + } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index faf266b63c67a..733e87f73cc23 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -173,8 +173,7 @@ public void testFailWriteFullMetadataNonClusterManagerNode() throws IOException public void testFailInitializationWhenRemoteStateDisabled() { final Settings settings = Settings.builder().build(); - when(clusterService.getClusterSettings()) - .thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); + when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); assertThrows( AssertionError.class, () -> new RemoteClusterStateService( From 1febb6816e3a628e7826b7c502961b1dc6c7d4be Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 30 Apr 2024 16:48:56 +0530 Subject: [PATCH 026/133] Modify the Integ test Signed-off-by: Shivansh Arora --- .../RemoteClusterStateCleanupManagerIT.java | 67 +++++++------------ .../RemoteClusterStateCleanupManager.java | 16 ++--- 2 files changed, 34 insertions(+), 49 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java index 033aa0e186539..4f995a5e44d5f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -26,6 +26,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; +import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) @@ -79,63 +80,47 @@ public void testRemoteCleanupTaskUpdated() { assertEquals(1, remoteClusterStateCleanupManager.getStaleFileDeletionTask().getInterval().getMinutes()); } - public void testRemoteCleanupOnlyAfter10Updates() throws Exception { + public void testRemoteCleanupDeleteStale() throws Exception { int shardCount = randomIntBetween(1, 2); - int replicaCount = 1; + int replicaCount = 3; int dataNodeCount = shardCount * (replicaCount + 1); int clusterManagerNodeCount = 1; initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); - // set cleanup interval to 100 ms + // set cleanup interval to 100 ms to make the test faster ClusterUpdateSettingsResponse response = client().admin() .cluster() .prepareUpdateSettings() .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "100ms")) .get(); - assertEquals(true, response.isAcknowledged()); - - replicaCount = updateReplicaCountNTimes(9, replicaCount); - - RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); - - BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); - BlobPath baseMetadataPath = repository.basePath() - .add( - Base64.getUrlEncoder() - .withoutPadding() - .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) - ) - .add("cluster-state") - .add(getClusterState().metadata().clusterUUID()); - BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); - - assertBusy(() -> { - assertEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); - }, 500, TimeUnit.MILLISECONDS); - - replicaCount = updateReplicaCountNTimes(8, replicaCount); - - // wait for 1 min, to ensure that clean up task ran and didn't clean up stale files because it was less than 10 - Thread.sleep(100); - assertNotEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); + assertTrue(response.isAcknowledged()); - // Do 2 more updates, now since the total successful state changes are more than 10, stale files will be cleaned up - replicaCount = updateReplicaCountNTimes(2, replicaCount); + // update replica count to simulate cluster state changes, so that we verify number of manifest files + replicaCount = updateReplicaCountNTimes(RETAINED_MANIFESTS + 2 * SKIP_CLEANUP_STATE_CHANGES, replicaCount); assertBusy(() -> { - assertEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); + BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); + BlobPath baseMetadataPath = repository.basePath() + .add( + Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) + ) + .add("cluster-state") + .add(getClusterState().metadata().clusterUUID()); + BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); + int manifestFiles = repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size(); + // we can't guarantee that we have same number of manifest as Retained manifest in our repo as there can be other queued task + // other than replica count change which can upload new manifest files, that's why we check that number of manifests is between + // Retained manifests and Retained manifests + Skip cleanup state changes + assertTrue( + "Current number of manifest files: " + manifestFiles, + manifestFiles >= RETAINED_MANIFESTS && manifestFiles <= RETAINED_MANIFESTS + SKIP_CLEANUP_STATE_CHANGES ); - }, 100, TimeUnit.MILLISECONDS); + }, 5000, TimeUnit.MILLISECONDS); RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( RemoteClusterStateService.class diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 0fc6c01231331..4f3f3b3bf3599 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -65,8 +65,8 @@ public class RemoteClusterStateCleanupManager implements Closeable { Setting.Property.Dynamic ); private static final Logger logger = LogManager.getLogger(RemoteClusterStateCleanupManager.class); - private RemoteClusterStateService remoteClusterStateService; - private RemotePersistenceStats remoteStateStats; + private final RemoteClusterStateService remoteClusterStateService; + private final RemotePersistenceStats remoteStateStats; private BlobStoreTransferService blobStoreTransferService; private volatile TimeValue staleFileCleanupInterval; private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); @@ -239,8 +239,8 @@ public void onResponse(List blobMetadata) { deleteClusterMetadata( clusterName, clusterUUID, - blobMetadata.subList(0, manifestsToRetain - 1), - blobMetadata.subList(manifestsToRetain - 1, blobMetadata.size()) + blobMetadata.subList(0, manifestsToRetain), + blobMetadata.subList(manifestsToRetain, blobMetadata.size()) ); } deleteStaleMetadataRunning.set(false); @@ -271,8 +271,8 @@ public void onFailure(Exception e) { * @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged */ void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUIDs) { - clusterUUIDs.forEach(clusterUUID -> { - getBlobStoreTransferService().deleteAsync( + clusterUUIDs.forEach( + clusterUUID -> getBlobStoreTransferService().deleteAsync( ThreadPool.Names.REMOTE_PURGE, remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), new ActionListener<>() { @@ -293,8 +293,8 @@ public void onFailure(Exception e) { remoteStateStats.cleanUpAttemptFailed(); } } - ); - }); + ) + ); } private void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { From 494aacc7c73671d76e298284fcbcee1a3072636f Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 30 Apr 2024 20:30:25 +0530 Subject: [PATCH 027/133] Address PR comments Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 97 ++++++++++--------- .../remote/RemoteClusterStateService.java | 47 +++++---- 2 files changed, 76 insertions(+), 68 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 023eb06b8eb86..de2e6727e9507 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -59,6 +59,56 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField UPLOADED_TEMPLATES_METADATA = new ParseField("uploaded_templates_metadata"); private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata"); + private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields) { + return ClusterMetadataManifest.builder() + .clusterTerm(term(fields)) + .stateVersion(version(fields)) + .clusterUUID(clusterUUID(fields)) + .stateUUID(stateUUID(fields)) + .opensearchVersion(opensearchVersion(fields)) + .nodeId(nodeId(fields)) + .committed(committed(fields)) + .codecVersion(CODEC_V0) + .indices(indices(fields)) + .previousClusterUUID(previousClusterUUID(fields)) + .clusterUUIDCommitted(clusterUUIDCommitted(fields)); + } + + private static ClusterMetadataManifest.Builder manifestV1Builder(Object[] fields) { + return ClusterMetadataManifest.builder() + .clusterTerm(term(fields)) + .stateVersion(version(fields)) + .clusterUUID(clusterUUID(fields)) + .stateUUID(stateUUID(fields)) + .opensearchVersion(opensearchVersion(fields)) + .nodeId(nodeId(fields)) + .committed(committed(fields)) + .codecVersion(codecVersion(fields)) + .globalMetadataFileName(globalMetadataFileName(fields)) + .indices(indices(fields)) + .previousClusterUUID(previousClusterUUID(fields)) + .clusterUUIDCommitted(clusterUUIDCommitted(fields)); + } + + private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields) { + return ClusterMetadataManifest.builder() + .clusterTerm(term(fields)) + .stateVersion(version(fields)) + .clusterUUID(clusterUUID(fields)) + .stateUUID(stateUUID(fields)) + .opensearchVersion(opensearchVersion(fields)) + .nodeId(nodeId(fields)) + .committed(committed(fields)) + .codecVersion(codecVersion(fields)) + .indices(indices(fields)) + .previousClusterUUID(previousClusterUUID(fields)) + .clusterUUIDCommitted(clusterUUIDCommitted(fields)) + .coordinationMetadata(coordinationMetadata(fields)) + .settingMetadata(settingsMetadata(fields)) + .templatesMetadata(templatesMetadata(fields)) + .customMetadataMap(customMetadata(fields)); + } + private static long term(Object[] fields) { return (long) fields[0]; } @@ -126,58 +176,17 @@ private static Map customMetadata(Object[] fi private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", - fields -> ClusterMetadataManifest.builder() - .clusterTerm(term(fields)) - .stateVersion(version(fields)) - .clusterUUID(clusterUUID(fields)) - .stateUUID(stateUUID(fields)) - .opensearchVersion(opensearchVersion(fields)) - .nodeId(nodeId(fields)) - .committed(committed(fields)) - .codecVersion(CODEC_V0) - .indices(indices(fields)) - .previousClusterUUID(previousClusterUUID(fields)) - .clusterUUIDCommitted(clusterUUIDCommitted(fields)) - .build() + fields -> manifestV0Builder(fields).build() ); private static final ConstructingObjectParser PARSER_V1 = new ConstructingObjectParser<>( "cluster_metadata_manifest", - fields -> ClusterMetadataManifest.builder() - .clusterTerm(term(fields)) - .stateVersion(version(fields)) - .clusterUUID(clusterUUID(fields)) - .stateUUID(stateUUID(fields)) - .opensearchVersion(opensearchVersion(fields)) - .nodeId(nodeId(fields)) - .committed(committed(fields)) - .codecVersion(codecVersion(fields)) - .globalMetadataFileName(globalMetadataFileName(fields)) - .indices(indices(fields)) - .previousClusterUUID(previousClusterUUID(fields)) - .clusterUUIDCommitted(clusterUUIDCommitted(fields)) - .build() + fields -> manifestV1Builder(fields).build() ); private static final ConstructingObjectParser PARSER_V2 = new ConstructingObjectParser<>( "cluster_metadata_manifest", - fields -> ClusterMetadataManifest.builder() - .clusterTerm(term(fields)) - .stateVersion(version(fields)) - .clusterUUID(clusterUUID(fields)) - .stateUUID(stateUUID(fields)) - .opensearchVersion(opensearchVersion(fields)) - .nodeId(nodeId(fields)) - .committed(committed(fields)) - .codecVersion(codecVersion(fields)) - .indices(indices(fields)) - .previousClusterUUID(previousClusterUUID(fields)) - .clusterUUIDCommitted(clusterUUIDCommitted(fields)) - .coordinationMetadata(coordinationMetadata(fields)) - .settingMetadata(settingsMetadata(fields)) - .templatesMetadata(templatesMetadata(fields)) - .customMetadataMap(customMetadata(fields)) - .build() + fields -> manifestV2Builder(fields).build() ); private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index cb66323be8994..fa3d5f3773685 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -210,6 +210,10 @@ public class RemoteClusterStateService implements Closeable { private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; + private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; + private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + + "updated : [{}], custom metadata updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; @@ -442,34 +446,29 @@ public ClusterMetadataManifest writeIncrementalMetadata( final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); + ParameterizedMessage clusterStateUploadTimeMessage = new ParameterizedMessage( + CLUSTER_STATE_UPLOAD_TIME_LOG_STRING, + manifest.getStateVersion(), + durationMillis + ); + ParameterizedMessage metadataUpdateMessage = new ParameterizedMessage( + METADATA_UPDATE_LOG_STRING, + numIndicesUpdated, + numIndicesUnchanged, + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata, + customsToUpload.size() + ); if (durationMillis >= slowWriteLoggingThreshold.getMillis()) { logger.warn( - "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " - + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " - + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", - durationMillis, + "{} which is above the warn threshold of [{}]; {}", + clusterStateUploadTimeMessage, slowWriteLoggingThreshold, - numIndicesUpdated, - numIndicesUnchanged, - updateCoordinationMetadata, - updateSettingsMetadata, - updateTemplatesMetadata, - customsToUpload.size() + metadataUpdateMessage ); } else { - logger.info( - "writing cluster state for version [{}] took [{}ms]; " - + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " - + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", - manifest.getStateVersion(), - durationMillis, - numIndicesUpdated, - numIndicesUnchanged, - updateCoordinationMetadata, - updateSettingsMetadata, - updateTemplatesMetadata, - customsToUpload.size() - ); + logger.info("{}; {}", clusterStateUploadTimeMessage, metadataUpdateMessage); } return manifest; } @@ -549,7 +548,7 @@ private UploadedMetadataResults writeMetadataInParallel( ); }); indexToUpload.forEach(indexMetadata -> { - uploadTasks.put(indexMetadata.getIndexName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); + uploadTasks.put(indexMetadata.getIndex().getName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); }); // start async upload of all required metadata files From 928b65036d200c15865f33994a88349875b7f6f0 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 2 May 2024 14:14:14 +0530 Subject: [PATCH 028/133] Address further PR comment Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index de2e6727e9507..d32c3787cdcfd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -75,34 +75,14 @@ private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields } private static ClusterMetadataManifest.Builder manifestV1Builder(Object[] fields) { - return ClusterMetadataManifest.builder() - .clusterTerm(term(fields)) - .stateVersion(version(fields)) - .clusterUUID(clusterUUID(fields)) - .stateUUID(stateUUID(fields)) - .opensearchVersion(opensearchVersion(fields)) - .nodeId(nodeId(fields)) - .committed(committed(fields)) + return manifestV0Builder(fields) .codecVersion(codecVersion(fields)) - .globalMetadataFileName(globalMetadataFileName(fields)) - .indices(indices(fields)) - .previousClusterUUID(previousClusterUUID(fields)) - .clusterUUIDCommitted(clusterUUIDCommitted(fields)); + .globalMetadataFileName(globalMetadataFileName(fields)); } private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields) { - return ClusterMetadataManifest.builder() - .clusterTerm(term(fields)) - .stateVersion(version(fields)) - .clusterUUID(clusterUUID(fields)) - .stateUUID(stateUUID(fields)) - .opensearchVersion(opensearchVersion(fields)) - .nodeId(nodeId(fields)) - .committed(committed(fields)) + return manifestV0Builder(fields) .codecVersion(codecVersion(fields)) - .indices(indices(fields)) - .previousClusterUUID(previousClusterUUID(fields)) - .clusterUUIDCommitted(clusterUUIDCommitted(fields)) .coordinationMetadata(coordinationMetadata(fields)) .settingMetadata(settingsMetadata(fields)) .templatesMetadata(templatesMetadata(fields)) From 2ebfc6de25614dc30604e469ed3f3d37df85ce9b Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 2 May 2024 14:42:11 +0530 Subject: [PATCH 029/133] apply spotless Signed-off-by: Shivansh Arora --- .../opensearch/gateway/remote/ClusterMetadataManifest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index d32c3787cdcfd..5c1510f96f0b9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -75,14 +75,11 @@ private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields } private static ClusterMetadataManifest.Builder manifestV1Builder(Object[] fields) { - return manifestV0Builder(fields) - .codecVersion(codecVersion(fields)) - .globalMetadataFileName(globalMetadataFileName(fields)); + return manifestV0Builder(fields).codecVersion(codecVersion(fields)).globalMetadataFileName(globalMetadataFileName(fields)); } private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields) { - return manifestV0Builder(fields) - .codecVersion(codecVersion(fields)) + return manifestV0Builder(fields).codecVersion(codecVersion(fields)) .coordinationMetadata(coordinationMetadata(fields)) .settingMetadata(settingsMetadata(fields)) .templatesMetadata(templatesMetadata(fields)) From 8f5d7d7f1977b9e7685521175648195b014c2c38 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 2 May 2024 16:22:14 +0530 Subject: [PATCH 030/133] Address PR comment Signed-off-by: Shivansh Arora --- .../RemoteClusterStateCleanupManagerIT.java | 78 ++++++++----------- .../remote/RemoteClusterStateService.java | 5 +- 2 files changed, 37 insertions(+), 46 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java index 4f995a5e44d5f..bf0db9dfe4141 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -28,6 +28,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteClusterStateCleanupManagerIT extends RemoteStoreBaseIntegTestCase { @@ -82,7 +83,7 @@ public void testRemoteCleanupTaskUpdated() { public void testRemoteCleanupDeleteStale() throws Exception { int shardCount = randomIntBetween(1, 2); - int replicaCount = 3; + int replicaCount = 1; int dataNodeCount = shardCount * (replicaCount + 1); int clusterManagerNodeCount = 1; @@ -97,61 +98,48 @@ public void testRemoteCleanupDeleteStale() throws Exception { assertTrue(response.isAcknowledged()); - // update replica count to simulate cluster state changes, so that we verify number of manifest files - replicaCount = updateReplicaCountNTimes(RETAINED_MANIFESTS + 2 * SKIP_CLEANUP_STATE_CHANGES, replicaCount); + // update cluster state 21 times to ensure that clean up has run after this will upload 42 manifest files + // to repository, if manifest files are less than that it means clean up has run + updateClusterStateNTimes(RETAINED_MANIFESTS + SKIP_CLEANUP_STATE_CHANGES + 1); + + RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); + BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); + BlobPath baseMetadataPath = repository.basePath() + .add( + Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) + ) + .add("cluster-state") + .add(getClusterState().metadata().clusterUUID()); + BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); assertBusy(() -> { - RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); - BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); - BlobPath baseMetadataPath = repository.basePath() - .add( - Base64.getUrlEncoder() - .withoutPadding() - .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) - ) - .add("cluster-state") - .add(getClusterState().metadata().clusterUUID()); - BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); int manifestFiles = repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size(); + logger.info("number of current manifest file: {}", manifestFiles); // we can't guarantee that we have same number of manifest as Retained manifest in our repo as there can be other queued task // other than replica count change which can upload new manifest files, that's why we check that number of manifests is between - // Retained manifests and Retained manifests + Skip cleanup state changes + // Retained manifests and Retained manifests + 2 * Skip cleanup state changes (each cluster state update uploads 2 manifests) assertTrue( "Current number of manifest files: " + manifestFiles, - manifestFiles >= RETAINED_MANIFESTS && manifestFiles <= RETAINED_MANIFESTS + SKIP_CLEANUP_STATE_CHANGES + manifestFiles >= RETAINED_MANIFESTS && manifestFiles < RETAINED_MANIFESTS + 2 * SKIP_CLEANUP_STATE_CHANGES ); - }, 5000, TimeUnit.MILLISECONDS); - - RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( - RemoteClusterStateService.class - ); - Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( - cluster().getClusterName(), - getClusterState().metadata().clusterUUID() - ).getMetadata().getIndices(); - assertEquals(replicaCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); - assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); - } - - private void setReplicaCount(int replicaCount) { - client().admin() - .indices() - .prepareUpdateSettings(INDEX_NAME) - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, replicaCount)) - .get(); - ensureGreen(INDEX_NAME); + }, 500, TimeUnit.MILLISECONDS); } - private int updateReplicaCountNTimes(int n, int initialCount) { + private void updateClusterStateNTimes(int n) { int newReplicaCount = randomIntBetween(0, 3); - ; - for (int i = 0; i < n; i++) { - while (newReplicaCount == initialCount) { - newReplicaCount = randomIntBetween(0, 3); - } - setReplicaCount(newReplicaCount); - initialCount = newReplicaCount; + for (int i = n; i>0; i--) { + ClusterUpdateSettingsResponse response = client().admin().cluster() + .prepareUpdateSettings() + .setPersistentSettings( + Settings.builder() + .put( + CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), + i, TimeUnit.SECONDS + ) + ).get(); + assertTrue(response.isAcknowledged()); } - return newReplicaCount; } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index d67a840b5a611..e2494d836d33a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -630,7 +630,10 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getGlobalMetadataFileName(), true ); - remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifest); + if (!previousManifest.isClusterUUIDCommitted() && committedManifest.isClusterUUIDCommitted()) { + remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifest); + } + return committedManifest; } From 4e3c9e92856247fe56242aea4ad3b7b876e68727 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 2 May 2024 16:26:52 +0530 Subject: [PATCH 031/133] Add changelog Signed-off-by: Shivansh Arora --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec85052873407..99dedc6639df4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225)) - [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386)) - [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276)) +- [Remote State] Add async remote state deletion task running on a interval, configurable by a setting ([#13131](https://github.com/opensearch-project/OpenSearch/pull/13131)) ### Dependencies - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896)) From 5ea5dbb565dee48ac276950afb7b5191e54bc8d1 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 2 May 2024 16:52:13 +0530 Subject: [PATCH 032/133] apply spotless Signed-off-by: Shivansh Arora --- .../RemoteClusterStateCleanupManagerIT.java | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java index bf0db9dfe4141..b9dac874ed0c0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerIT.java @@ -9,7 +9,6 @@ package org.opensearch.gateway.remote; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; @@ -22,7 +21,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; @@ -129,16 +127,12 @@ public void testRemoteCleanupDeleteStale() throws Exception { private void updateClusterStateNTimes(int n) { int newReplicaCount = randomIntBetween(0, 3); - for (int i = n; i>0; i--) { - ClusterUpdateSettingsResponse response = client().admin().cluster() + for (int i = n; i > 0; i--) { + ClusterUpdateSettingsResponse response = client().admin() + .cluster() .prepareUpdateSettings() - .setPersistentSettings( - Settings.builder() - .put( - CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), - i, TimeUnit.SECONDS - ) - ).get(); + .setPersistentSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), i, TimeUnit.SECONDS)) + .get(); assertTrue(response.isAcknowledged()); } } From fb0b6aaaed7a53f4c6811db6f78c0f0e7f42e644 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 3 May 2024 21:02:36 +0530 Subject: [PATCH 033/133] removed unnecessary method Signed-off-by: Shivansh Arora --- .../java/org/opensearch/cluster/metadata/IndexMetadata.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index eae59f0049a66..ec2b69bbda247 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -753,9 +753,6 @@ public String getIndexUUID() { return index.getUUID(); } - public String getIndexName() { - return index.getName(); - } /** * Test whether the current index UUID is the same as the given one. Returns true if either are _na_ From cae6632fae7199d1a9a500ff05a8980d5f85e2aa Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 6 May 2024 13:23:46 +0530 Subject: [PATCH 034/133] Input stream fixes --- .../remote/RemoteRoutingTableService.java | 75 +++++++++++++++++-- .../remote/ClusterMetadataManifest.java | 28 ++++--- .../remote/RemoteClusterStateService.java | 51 ++++++++++++- .../IndexRoutingTableInputStream.java | 31 ++++++-- 4 files changed, 163 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 7778353a75047..43fe2186dea09 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,23 +10,32 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.Version; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStream; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import java.io.Closeable; -import java.io.IOException; +import java.io.*; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; @@ -62,14 +71,70 @@ public RemoteRoutingTableService(Supplier repositoriesServi } public List writeFullRoutingTable(ClusterState clusterState, String previousClusterUUID) { - return null; + //batch index count and parallelize + RoutingTable currentRoutingTable = clusterState.getRoutingTable(); + List uploadedIndices = new ArrayList<>(); + for(IndexRoutingTable indexRouting: currentRoutingTable.getIndicesRouting().values()){ + try { + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting, currentRoutingTable.version(), Version.CURRENT); + BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add("routing-table")); + container.writeBlobWithMetadata(indexRouting.getIndex().getName(), indexRoutingStream, indexRoutingStream.read(), true, null); + logger.info("SUccessful write"); + uploadedIndices.add(new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), "dummyfilename")); + }catch (IOException e) { + logger.error("Failed to write {}", e); + } + } + logger.info("uploadedIndices {}", uploadedIndices); + + return uploadedIndices; } - public List writeIncrementalMetadata( + public List writeIncrementalRoutingTable( ClusterState previousClusterState, ClusterState clusterState, ClusterMetadataManifest previousManifest) { - return null; + final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() + .stream() + .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); + logger.info("allUploadedIndicesRouting ROUTING {}", allUploadedIndicesRouting); + + List uploadedIndices = new ArrayList<>(); + for(IndexRoutingTable indexRouting: clusterState.getRoutingTable().getIndicesRouting().values()){ + if(previousClusterState.getRoutingTable().getIndicesRouting().containsKey(indexRouting.getIndex().getName())) { + logger.info("index exists {}", indexRouting.getIndex().getName()); + //existing index, check if shards are changed + + IndexRoutingTable previousIndexRouting = previousClusterState.getRoutingTable().getIndicesRouting().get(indexRouting.getIndex().getName()); + if (indexRouting.equals(previousIndexRouting)){ + uploadedIndices.add(allUploadedIndicesRouting.get(indexRouting.getIndex().getName())); + continue; + } + try { + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting, clusterState.getRoutingTable().version(), Version.CURRENT); + BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add("routing-table").add(indexRouting.getIndex().getName()).add(String.valueOf(clusterState.getRoutingTable().version()))); + container.writeBlob(indexRouting.getIndex().getName(), indexRoutingStream, 4096,true); + logger.info("SUccessful write"); + uploadedIndices.add(new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString())); + } catch (IOException e) { + logger.error("Failed to write {}", e); + } + } else { + // new index upload + logger.info("cuurent version {}, previous sversiion {}", clusterState.getRoutingTable().version(), previousClusterState.getRoutingTable().version()); + try { + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting, clusterState.getRoutingTable().version(), Version.CURRENT); + BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add("routing-table").add(indexRouting.getIndex().getName()).add(String.valueOf(clusterState.getRoutingTable().version()))); + container.writeBlob(indexRouting.getIndex().getName(), indexRoutingStream, 4096,true); + //container.writeBlobWithMetadata(indexRouting.getIndex().getName(), indexRoutingStream, indexRoutingStream.read(), true, null); + logger.info("SUccessful write"); + uploadedIndices.add(new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString())); + } catch (IOException e) { + logger.error("Failed to write {}", e); + } + } + } + return uploadedIndices; } public RoutingTable getLatestRoutingTable(String clusterName, String clusterUUID) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4e5891d154de0..3222e44736bd4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -8,12 +8,15 @@ package org.opensearch.gateway.remote; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentFragment; @@ -21,10 +24,7 @@ import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; +import java.util.*; /** * Manifest file which contains the details of the uploaded entity metadata @@ -260,6 +260,7 @@ public String getGlobalMetadataFileName() { public List getIndicesRouting() { return indicesRouting; } + private static final Logger logger = LogManager.getLogger(ClusterMetadataManifest.class); public ClusterMetadataManifest( long clusterTerm, @@ -276,7 +277,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted ) { this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, null); + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, new ArrayList<>()); } public ClusterMetadataManifest( @@ -320,6 +321,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); + logger.info("VERSION {}", in.getVersion()); if (in.getVersion().onOrAfter(Version.V_2_14_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); @@ -327,7 +329,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); - this.indicesRouting = null; + this.indicesRouting =null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; @@ -345,6 +347,7 @@ public static Builder builder(ClusterMetadataManifest manifest) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + logger.info("CLUSTER_TERM_FIELD {} : {}", CLUSTER_TERM_FIELD.getPreferredName(), getClusterTerm()); builder.field(CLUSTER_TERM_FIELD.getPreferredName(), getClusterTerm()) .field(STATE_VERSION_FIELD.getPreferredName(), getStateVersion()) .field(CLUSTER_UUID_FIELD.getPreferredName(), getClusterUUID()) @@ -355,7 +358,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(INDICES_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indices) { + builder.startObject(); uploadedIndexMetadata.toXContent(builder, params); + builder.endObject(); } } builder.endArray(); @@ -369,9 +374,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { + builder.startObject(); uploadedIndexMetadata.toXContent(builder, params); + builder.endObject(); } } + builder.endArray(); } return builder; } @@ -391,7 +399,8 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); - } else if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + } + if (out.getVersion().onOrAfter(Version.V_2_14_0)) { out.writeCollection(indicesRouting); } } @@ -659,11 +668,10 @@ public String getIndexUUID() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() + return builder .field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID()) - .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()) - .endObject(); + .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 69214d6b22a61..e98df29e06467 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -174,7 +174,7 @@ public class RemoteClusterStateService implements Closeable { private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V1; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; // ToXContent Params with gateway mode. @@ -195,6 +195,8 @@ public RemoteClusterStateService( ThreadPool threadPool ) { assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled"; + logger.info("REMOTE STATE ENABLED"); + this.nodeId = nodeId; this.repositoriesService = repositoriesService; this.settings = settings; @@ -213,6 +215,7 @@ public RemoteClusterStateService( if(isRemoteRoutingTableEnabled(settings)) { this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, settings, clusterSettings); + logger.info("REMOTE ROUTING ENABLED"); } } @@ -231,6 +234,7 @@ private BlobStoreTransferService getBlobStoreTransferService() { */ @Nullable public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, String previousClusterUUID) throws IOException { + logger.info("WRITING FULL STATE"); final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { logger.error("Local node is not elected cluster manager. Exiting"); @@ -240,15 +244,24 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri // TODO: we can upload global metadata and index metadata in parallel. [issue: #10645] // Write globalMetadata String globalMetadataFile = writeGlobalMetadata(clusterState); + logger.info("globalMetadataFile {}", globalMetadataFile); // any validations before/after upload ? final List allUploadedIndexMetadata = writeIndexMetadataParallel( clusterState, new ArrayList<>(clusterState.metadata().indices().values()) ); + + List routingIndexMetadata = new ArrayList<>(); + if(remoteRoutingTableService!=null) { + routingIndexMetadata = remoteRoutingTableService.writeFullRoutingTable(clusterState, previousClusterUUID); + logger.info("routingIndexMetadata {}", routingIndexMetadata); + } + final ClusterMetadataManifest manifest = uploadManifest( clusterState, allUploadedIndexMetadata, + routingIndexMetadata, previousClusterUUID, globalMetadataFile, false @@ -286,6 +299,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( ClusterState clusterState, ClusterMetadataManifest previousManifest ) throws IOException { + logger.info("WRITING INCREMENTAL STATE"); + final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { logger.error("Local node is not elected cluster manager. Exiting"); @@ -313,12 +328,14 @@ public ClusterMetadataManifest writeIncrementalMetadata( for (final IndexMetadata indexMetadata : previousClusterState.metadata().indices().values()) { previousStateIndexMetadataVersionByName.put(indexMetadata.getIndex().getName(), indexMetadata.getVersion()); } + logger.info("previousStateIndexMetadataVersionByName incremental {}", previousStateIndexMetadataVersionByName); int numIndicesUpdated = 0; int numIndicesUnchanged = 0; final Map allUploadedIndexMetadata = previousManifest.getIndices() .stream() .collect(Collectors.toMap(UploadedIndexMetadata::getIndexName, Function.identity())); + logger.info("allUploadedIndexMetadata incremental {}", allUploadedIndexMetadata); List toUpload = new ArrayList<>(); @@ -338,18 +355,30 @@ public ClusterMetadataManifest writeIncrementalMetadata( } previousStateIndexMetadataVersionByName.remove(indexMetadata.getIndex().getName()); } + logger.info("toUpload incremental {}", toUpload); + logger.info("previousStateIndexMetadataVersionByName after rmeove {}", previousStateIndexMetadataVersionByName); List uploadedIndexMetadataList = writeIndexMetadataParallel(clusterState, toUpload); uploadedIndexMetadataList.forEach( uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata) ); + logger.info("uploadedIndexMetadataList incremental {}", uploadedIndexMetadataList); for (String removedIndexName : previousStateIndexMetadataVersionByName.keySet()) { allUploadedIndexMetadata.remove(removedIndexName); } + logger.info("allUploadedIndexMetadata after rmeove {}", uploadedIndexMetadataList); + + List routingIndexMetadata = new ArrayList<>(); + if(remoteRoutingTableService!=null) { + routingIndexMetadata = remoteRoutingTableService.writeIncrementalRoutingTable(previousClusterState, clusterState, previousManifest); + logger.info("routingIndexMetadata incremental {}", routingIndexMetadata); + } + final ClusterMetadataManifest manifest = uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), + routingIndexMetadata, previousManifest.getPreviousClusterUUID(), globalMetadataFile, false @@ -562,9 +591,11 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat return null; } assert previousManifest != null : "Last cluster metadata manifest is not set"; + logger.info("MARKING LAST STATE COMMIITTED"); ClusterMetadataManifest committedManifest = uploadManifest( clusterState, previousManifest.getIndices(), + previousManifest.getIndicesRouting(), previousManifest.getPreviousClusterUUID(), previousManifest.getGlobalMetadataFileName(), true @@ -603,9 +634,21 @@ private ClusterMetadataManifest uploadManifest( String previousClusterUUID, String globalClusterMetadataFileName, boolean committed + ) throws IOException { + return uploadManifest(clusterState, uploadedIndexMetadata, new ArrayList<>(), previousClusterUUID, globalClusterMetadataFileName, committed); + } + + private ClusterMetadataManifest uploadManifest( + ClusterState clusterState, + List uploadedIndexMetadata, + List uploadedIndicesRouting, + String previousClusterUUID, + String globalClusterMetadataFileName, + boolean committed ) throws IOException { synchronized (this) { final String manifestFileName = getManifestFileName(clusterState.term(), clusterState.version(), committed); + logger.info("manifestFileName {}", manifestFileName); final ClusterMetadataManifest manifest = new ClusterMetadataManifest( clusterState.term(), clusterState.getVersion(), @@ -618,8 +661,11 @@ private ClusterMetadataManifest uploadManifest( globalClusterMetadataFileName, uploadedIndexMetadata, previousClusterUUID, - clusterState.metadata().clusterUUIDCommitted() + clusterState.metadata().clusterUUIDCommitted(), + uploadedIndicesRouting ); + logger.info("manifest {}", manifest); + writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; } @@ -639,6 +685,7 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); }, ex -> { exceptionReference.set(ex); }), latch); + logger.info("MANIFEST {}", uploadManifest); CLUSTER_METADATA_MANIFEST_FORMAT.writeAsyncWithUrgentPriority( uploadManifest, metadataManifestContainer, diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index ac65232e9a24d..4918498edb854 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -8,11 +8,14 @@ package org.opensearch.gateway.remote.routingtable; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; @@ -56,6 +59,7 @@ public class IndexRoutingTableInputStream extends InputStream { private final IndexRoutingTableHeader indexRoutingTableHeader; private final Iterator shardIter; + private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStream.class); public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); @@ -66,6 +70,8 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long ve this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + logger.info("indexRoutingTable {}, version {}, nodeVersion {}", indexRoutingTable.prettyPrint(), version, nodeVersion); + initialFill(); } @@ -82,12 +88,26 @@ private void initialFill() throws IOException { BytesReference bytesReference = indexRoutingTableHeader.write(); buf = bytesReference.toBytesRef().bytes; count = bytesReference.length(); + logger.info("bytesReference {} buf {}, count {}", bytesReference , buf, count); + fill(buf); } private void fill(byte[] buf) throws IOException { if (leftOverBuf != null) { - System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + if(leftOverBuf.length > buf.length - count) { + // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in leftOverBuf. + System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); + byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; + System.arraycopy(leftOverBuf, buf.length - count , tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); + leftOverBuf = tempLeftOverBuffer; + count = buf.length - count; + + } else { + System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); + count += leftOverBuf.length; + leftOverBuf = null; + } } if (count < buf.length && shardIter.hasNext()) { IndexShardRoutingTable next = shardIter.next(); @@ -108,9 +128,10 @@ private void fill(byte[] buf) throws IOException { leftOverBuf = null; } else { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); - count += buf.length - count; - leftOverBuf = new byte[bytesRef.length() - count]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count + 1, leftOverBuf, 0, bytesRef.length() - count); + leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count , leftOverBuf, 0, bytesRef.length() - (buf.length - count)); + count = buf.length; + } } } @@ -128,7 +149,7 @@ else if (pos >= buffer.length) { /* no room left in buffer */ markPos = -1; /* buffer got too big, invalidate mark */ pos = 0; /* drop buffer contents */ } else { /* grow buffer */ - int nsz = markLimit + 1; + int nsz = markLimit + 1; //NEED TO CHECK THIS byte[] nbuf = new byte[nsz]; System.arraycopy(buffer, 0, nbuf, 0, pos); buffer = nbuf; From 40fec5fed35611cc2140ef2c5ab1eef7ee166f03 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 6 May 2024 17:52:34 +0530 Subject: [PATCH 035/133] Checksum calculation at end of file --- .../routingtable/IndexRoutingTableHeader.java | 15 ++------ .../IndexRoutingTableInputStream.java | 36 +++++++++++-------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 04a0f1868b64f..2a26f10ac4cf0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -21,6 +21,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.EOFException; import java.io.IOException; @@ -50,26 +51,16 @@ public IndexRoutingTableHeader(long routingTableVersion, String indexName, Versi /** * Returns the bytes reference for the {@link IndexRoutingTableHeader} - * @return the {@link BytesReference} * @throws IOException */ - public BytesReference write() throws IOException { - BytesReference bytesReference; - try ( - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) - ) { + public void write(StreamOutput out) throws IOException { CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); // Write version out.writeLong(routingTableVersion); out.writeInt(nodeVersion.id); out.writeString(indexName); - // Checksum header - out.writeInt((int) out.getChecksum()); + out.flush(); - bytesReference = bytesStreamOutput.bytes(); - } - return bytesReference; } /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index 4918498edb854..0b7f2cbb2326c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -60,6 +60,8 @@ public class IndexRoutingTableInputStream extends InputStream { private final Iterator shardIter; private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStream.class); + private final BytesStreamOutput bytesStreamOutput; + private final BufferedChecksumStreamOutput out; public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); @@ -69,10 +71,13 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long ve throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); + this.bytesStreamOutput = new BytesStreamOutput(); + this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + logger.info("indexRoutingTable {}, version {}, nodeVersion {}", indexRoutingTable.prettyPrint(), version, nodeVersion); - initialFill(); + initialFill(indexRoutingTable.shards().size()); } @Override @@ -84,12 +89,13 @@ public int read() throws IOException { return buf[pos++] & 0xff; } - private void initialFill() throws IOException { - BytesReference bytesReference = indexRoutingTableHeader.write(); - buf = bytesReference.toBytesRef().bytes; - count = bytesReference.length(); - logger.info("bytesReference {} buf {}, count {}", bytesReference , buf, count); + private void initialFill(int shardCount) throws IOException { + indexRoutingTableHeader.write(out); + out.writeVInt(shardCount); + System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0 , buf, 0, bytesStreamOutput.bytes().length()); + count = bytesStreamOutput.bytes().length(); + bytesStreamOutput.reset(); fill(buf); } @@ -109,19 +115,18 @@ private void fill(byte[] buf) throws IOException { leftOverBuf = null; } } + if (count < buf.length && shardIter.hasNext()) { IndexShardRoutingTable next = shardIter.next(); - BytesReference bytesRef; - try ( - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - BufferedChecksumStreamOutput out = new BufferedChecksumStreamOutput(bytesStreamOutput) - ) { - IndexShardRoutingTable.Builder.writeTo(next, out); - // Checksum header + IndexShardRoutingTable.Builder.writeTo(next, out); + //Add checksum for the file after all shards are done + if(!shardIter.hasNext()) { out.writeInt((int) out.getChecksum()); - out.flush(); - bytesRef = bytesStreamOutput.bytes(); } + out.flush(); + BytesReference bytesRef = bytesStreamOutput.bytes(); + bytesStreamOutput.reset(); + if (bytesRef.length() < buf.length - count) { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, bytesRef.length()); count += bytesRef.length(); @@ -134,6 +139,7 @@ private void fill(byte[] buf) throws IOException { } } + } private void maybeResizeAndFill() throws IOException { From d5fc14769391084734246c2209961ef77e517048 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 7 May 2024 15:31:37 +0530 Subject: [PATCH 036/133] Add toXContent and fromXContent for DiscoveryNode and DiscoveryNodes Signed-off-by: Shivansh Arora --- .../common/transport/TransportAddress.java | 10 + .../cluster/node/DiscoveryNode.java | 96 +++++++++- .../cluster/node/DiscoveryNodes.java | 69 ++++++- .../cluster/node/DiscoveryNodeTests.java | 74 ++++++++ .../cluster/node/DiscoveryNodesTests.java | 177 ++++++++++++++++++ 5 files changed, 421 insertions(+), 5 deletions(-) diff --git a/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java b/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java index 3b5fbb7d76307..b7ab988ce0404 100644 --- a/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java +++ b/libs/core/src/main/java/org/opensearch/core/common/transport/TransportAddress.java @@ -162,4 +162,14 @@ public String toString() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.value(toString()); } + + public static TransportAddress fromString(String address) throws UnknownHostException { + String[] addressSplit = address.split(":"); + if (addressSplit.length != 2) { + throw new IllegalArgumentException("address must be of the form [hostname/ip]:[port]"); + } + String hostname = addressSplit[0]; + int port = Integer.parseInt(addressSplit[1]); + return new TransportAddress(InetAddress.getByName(hostname), port); + } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 5226e9570ac14..644e5f3de9352 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -33,6 +33,7 @@ package org.opensearch.cluster.node; import org.opensearch.Version; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; @@ -43,6 +44,7 @@ import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.node.Node; import java.io.IOException; @@ -60,6 +62,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_API; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_PARAM; import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; @@ -72,6 +76,14 @@ public class DiscoveryNode implements Writeable, ToXContentFragment { static final String COORDINATING_ONLY = "coordinating_only"; + static final String KEY_NAME = "name"; + static final String KEY_EPHEMERAL_ID = "ephemeral_id"; + static final String KEY_HOST_NAME = "host_name"; + static final String KEY_HOST_ADDRESS = "host_address"; + static final String KEY_TRANSPORT_ADDRESS = "transport_address"; + static final String KEY_ATTRIBUTES = "attributes"; + static final String KEY_VERSION = "version"; + static final String KEY_ROLES = "roles"; public static boolean nodeRequiresLocalStorage(Settings settings) { boolean localStorageEnable = Node.NODE_LOCAL_STORAGE_SETTING.get(settings); @@ -544,21 +556,97 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Metadata.XContentContext context = Metadata.XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API)); builder.startObject(getId()); - builder.field("name", getName()); - builder.field("ephemeral_id", getEphemeralId()); - builder.field("transport_address", getAddress().toString()); + builder.field(KEY_NAME, getName()); + builder.field(KEY_EPHEMERAL_ID, getEphemeralId()); + builder.field(KEY_TRANSPORT_ADDRESS, getAddress().toString()); - builder.startObject("attributes"); + builder.startObject(KEY_ATTRIBUTES); for (Map.Entry entry : attributes.entrySet()) { builder.field(entry.getKey(), entry.getValue()); } builder.endObject(); + if (context == Metadata.XContentContext.GATEWAY) { + builder.field(KEY_HOST_NAME, getHostName()); + builder.field(KEY_HOST_ADDRESS, getHostAddress()); + builder.field(KEY_VERSION, getVersion().toString()); + builder.startArray(KEY_ROLES); + for (DiscoveryNodeRole role : roles) { + builder.value(role.roleName()); + } + builder.endArray(); + } builder.endObject(); return builder; } + public static DiscoveryNode fromXContent(XContentParser parser, String nodeId) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + if (parser.currentToken() != XContentParser.Token.FIELD_NAME) { + throw new IllegalArgumentException("expected field name but got a " + parser.currentToken()); + } + String nodeName = null; + String hostName = null; + String hostAddress = null; + String ephemeralId = null; + TransportAddress transportAddress = null; + Map attributes = new HashMap<>(); + Set roles = new HashSet<>(); + Version version = null; + String currentFieldName = parser.currentName(); + // token should be start object at this point + // XContentParser.Token token = parser.nextToken(); + // if (token != XContentParser.Token.START_OBJECT) { + // throw new IllegalArgumentException("expected object but got a " + token); + // } + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (KEY_NAME.equals(currentFieldName)) { + nodeName = parser.text(); + } else if (KEY_EPHEMERAL_ID.equals(currentFieldName)) { + ephemeralId = parser.text(); + } else if (KEY_TRANSPORT_ADDRESS.equals(currentFieldName)) { + transportAddress = TransportAddress.fromString(parser.text()); + } else if (KEY_HOST_NAME.equals(currentFieldName)) { + hostName = parser.text(); + } else if (KEY_HOST_ADDRESS.equals(currentFieldName)) { + hostAddress = parser.text(); + } else if (KEY_VERSION.equals(currentFieldName)) { + version = Version.fromString(parser.text()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (KEY_ATTRIBUTES.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + attributes.put(currentFieldName, parser.text()); + } + } + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (KEY_ROLES.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + roles.add(getRoleFromRoleName(parser.text())); + } + } + } else { + throw new IllegalArgumentException("unexpected token " + token); + } + } + return new DiscoveryNode(nodeName, nodeId, ephemeralId, hostName, hostAddress, transportAddress, attributes, roles, version); + } + private static Map rolesToMap(final Stream roles) { return Collections.unmodifiableMap(roles.collect(Collectors.toMap(DiscoveryNodeRole::roleName, Function.identity()))); } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index 2ebcd8096893d..d281021624fd0 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -35,6 +35,7 @@ import org.opensearch.Version; import org.opensearch.cluster.AbstractDiffable; import org.opensearch.cluster.Diff; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; @@ -44,6 +45,9 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; @@ -59,6 +63,9 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_API; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_PARAM; + /** * This class holds all {@link DiscoveryNode} in the cluster and provides convenience methods to * access, modify merge / diff discovery nodes. @@ -66,7 +73,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class DiscoveryNodes extends AbstractDiffable implements Iterable { +public class DiscoveryNodes extends AbstractDiffable implements Iterable, ToXContentFragment { public static final DiscoveryNodes EMPTY_NODES = builder().build(); @@ -566,6 +573,66 @@ public String toString() { return sb.toString(); } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("nodes"); + for (DiscoveryNode node : this) { + node.toXContent(builder, params); + } + builder.endObject(); + Metadata.XContentContext context = Metadata.XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API)); + if (context == Metadata.XContentContext.GATEWAY && clusterManagerNodeId != null) { + builder.field("cluster_manager", clusterManagerNodeId); + } + return builder; + } + + public static DiscoveryNodes fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + if (parser.currentToken() == null) { + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + if (parser.currentToken() != XContentParser.Token.FIELD_NAME) { + throw new IllegalArgumentException("expected field name but got a " + parser.currentToken()); + } + XContentParser.Token token; + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if ("nodes".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + String nodeId = currentFieldName; + DiscoveryNode node = DiscoveryNode.fromXContent(parser, nodeId); + builder.add(node); + } + } + } else { + throw new IllegalArgumentException("unexpected object field " + currentFieldName); + } + } else if (token.isValue()) { + if ("cluster_manager".equals(currentFieldName)) { + String clusterManagerNodeId = parser.text(); + if (clusterManagerNodeId != null) { + builder.clusterManagerNodeId(clusterManagerNodeId); + } + } else { + throw new IllegalArgumentException("unexpected value field " + currentFieldName); + } + } else { + throw new IllegalArgumentException("unexpected token " + token); + } + } + return builder.build(); + } + /** * Delta between nodes. * diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index c8a6fc76ce820..b3a4c7e34125e 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -33,15 +33,20 @@ package org.opensearch.cluster.node; import org.opensearch.Version; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.test.NodeRoles; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.net.InetAddress; import java.util.Collections; import java.util.HashMap; @@ -53,6 +58,9 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static java.util.Collections.singletonMap; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_API; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_GATEWAY; import static org.opensearch.test.NodeRoles.nonRemoteClusterClientNode; import static org.opensearch.test.NodeRoles.nonSearchNode; import static org.opensearch.test.NodeRoles.remoteClusterClientNode; @@ -249,4 +257,70 @@ public void testDiscoveryNodeIsSearchNode() { final DiscoveryNode node = DiscoveryNode.createLocal(settingWithSearchRole, buildNewFakeTransportAddress(), "node"); assertThat(node.isSearchNode(), equalTo(true)); } + + public void testToXContentInAPIMode() throws IOException { + final DiscoveryNode node = DiscoveryNode.createLocal( + Settings.EMPTY, + new TransportAddress(TransportAddress.META_ADDRESS, 9200), + "node_1" + ); + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + builder.startObject(); + node.toXContent(builder, new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_API))); + builder.endObject(); + + String expectedNodeAPUXContent = "{\n" + + " \"node_1\" : {\n" + + " \"name\" : \"" + + node.getName() + + "\",\n" + + " \"ephemeral_id\" : \"" + + node.getEphemeralId() + + "\",\n" + + " \"transport_address\" : \"0.0.0.0:9200\",\n" + + " \"attributes\" : { }\n" + + " }\n" + + "}"; + + assertEquals(expectedNodeAPUXContent, builder.toString()); + } + + public void testToXContentInGatewayMode() throws IOException { + final DiscoveryNode node = DiscoveryNode.createLocal( + Settings.EMPTY, + new TransportAddress(TransportAddress.META_ADDRESS, 9200), + "node_1" + ); + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + builder.startObject(); + node.toXContent(builder, new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY))); + builder.endObject(); + + String expectedNodeAPUXContent = "{\n" + + " \"node_1\" : {\n" + + " \"name\" : \"" + + node.getName() + + "\",\n" + + " \"ephemeral_id\" : \"" + + node.getEphemeralId() + + "\",\n" + + " \"transport_address\" : \"0.0.0.0:9200\",\n" + + " \"attributes\" : { },\n" + + " \"host_name\" : \"0.0.0.0\",\n" + + " \"host_address\" : \"0.0.0.0\",\n" + + " \"version\" : \"" + + node.getVersion() + + "\",\n" + + " \"roles\" : [\n" + + " \"cluster_manager\",\n" + + " \"data\",\n" + + " \"ingest\",\n" + + " \"remote_cluster_client\"\n" + + " ]\n" + + " }\n" + + "}"; + + assertEquals(expectedNodeAPUXContent, builder.toString()); + + } } diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java index d2450859dfcd4..3097af64a439f 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java @@ -36,10 +36,20 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.settings.Setting; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.XContentTestUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -47,13 +57,18 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.StreamSupport; +import static java.util.Collections.singletonMap; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_API; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_GATEWAY; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; @@ -497,6 +512,168 @@ public void testMaxMinNodeVersion() { assertEquals(LegacyESVersion.fromString("5.1.0"), build.getMinNodeVersion()); } + public void testToXContentInAPIMode() throws IOException { + DiscoveryNodes nodes = buildDiscoveryNodes(); + + String expectedNodeAPUXContent = "%1$s\"node_%2$d\" : {\n" + + "%1$s \"name\" : \"name_%2$d\",\n" + + "%1$s \"ephemeral_id\" : \"%3$s\",\n" + + "%1$s \"transport_address\" : \"0.0.0.0:%4$d\",\n" + + "%1$s \"attributes\" : {%5$s}\n" + + "%1$s}"; + + verifyToXContentInContextMode( + CONTEXT_MODE_API, + nodes, + "{\n" + + " \"nodes\" : {\n" + + IntStream.range(0, nodes.getSize()) + .mapToObj( + i -> String.format( + Locale.ROOT, + expectedNodeAPUXContent, + " ", + i, + nodes.get("node_" + i).getEphemeralId(), + i + 1, + nodes.get("node_" + i).getAttributes().isEmpty() + ? " " + : "\n" + " \"custom\" : \"" + nodes.get("node_" + i).getAttributes().get("custom") + "\"\n " + ) + ) + .collect(Collectors.joining(",\n")) + + "\n" + + " }\n" + + "}" + ); + } + + public void testXContentInGatewayMode() throws IOException { + DiscoveryNodes nodes = buildDiscoveryNodes(); + String expectedXContent = getExpectedXContentInGatewayMode(nodes); + + verifyToXContentInContextMode(CONTEXT_MODE_GATEWAY, nodes, expectedXContent); + } + + private String getExpectedXContentInGatewayMode(DiscoveryNodes nodes) { + /* + * Following formatting creates a string like following: + * "node_1" : { + * "name" : "name_1", + * "ephemeral_id" : "3Q3xRwYKScWqBgVCrWmNCQ", + * "transport_address" : "0.0.0.0:2", + * "attributes" : { + * "custom" : "PKU" + * }, + * "host_name" : "0.0.0.0", + * "host_address" : "0.0.0.0", + * "version" : "3.0.0", + * "roles" : [ + * "custom_role", + * "ingest", + * "remote_cluster_client", + * "search" + * ] + * } + * */ + String expectedNodeAPUXContent = "%1$s\"node_%2$d\" : {\n" + + "%1$s \"name\" : \"name_%2$d\",\n" + + "%1$s \"ephemeral_id\" : \"%3$s\",\n" + + "%1$s \"transport_address\" : \"0.0.0.0:%4$d\",\n" + + "%1$s \"attributes\" : {%5$s},\n" + + "%1$s \"host_name\" : \"0.0.0.0\",\n" + + "%1$s \"host_address\" : \"0.0.0.0\",\n" + + "%1$s \"version\" : \"%6$s\",\n" + + "%1$s \"roles\" : [%7$s]\n" + + "%1$s}"; + + return "{\n" + " \"nodes\" : {\n" + IntStream.range(0, nodes.getSize()).mapToObj(i -> { + String nodeId = "node_" + i; + DiscoveryNode node = nodes.get(nodeId); + String indent = " "; + return String.format( + Locale.ROOT, + expectedNodeAPUXContent, + indent, + i, + node.getEphemeralId(), + i + 1, + node.getAttributes().isEmpty() + ? " " + : "\n" + indent + " \"custom\" : \"" + nodes.get("node_" + i).getAttributes().get("custom") + "\"\n " + indent, + node.getVersion(), + node.getRoles().isEmpty() + ? " " + : "\n" + + indent + + " \"" + + node.getRoles().stream().map(DiscoveryNodeRole::roleName).collect(Collectors.joining("\",\n" + indent + " \"")) + + "\"\n " + + indent + ); + }).collect(Collectors.joining(",\n")) + + "\n" + + " },\n" + + " \"cluster_manager\" : \"" + + nodes.getClusterManagerNodeId() + + "\"\n" + + "}"; + } + + public void verifyToXContentInContextMode(String context, DiscoveryNodes nodes, String expected) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + builder.startObject(); + nodes.toXContent(builder, new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, context))); + builder.endObject(); + + assertEquals(expected, builder.toString()); + } + + public void testFromXContent() throws IOException { + doFromXContentTestWithRandomFields(false); + } + + public void testFromXContentWithRandomFields() throws IOException { + doFromXContentTestWithRandomFields(true); + } + + private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { + DiscoveryNodes nodes = buildDiscoveryNodes(); + boolean humanReadable = randomBoolean(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + nodes, + mediaType, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY)), + humanReadable + ); + + if (addRandomFields) { + String unsupportedField = "unsupported_field"; + BytesReference mutated = BytesReference.bytes( + XContentTestUtils.insertIntoXContent( + mediaType.xContent(), + originalBytes, + Collections.singletonList(""), + () -> unsupportedField, + () -> randomAlphaOfLengthBetween(3, 10) + ) + ); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> DiscoveryNodes.fromXContent(createParser(mediaType.xContent(), mutated)) + ); + assertEquals(iae.getMessage(), "unexpected value field " + unsupportedField); + } else { + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + DiscoveryNodes parsedNodes = DiscoveryNodes.fromXContent(parser); + assertEquals(nodes.getSize(), parsedNodes.getSize()); + nodes.forEach(node -> node.equals(parsedNodes.get(node.getId()))); + assertEquals(nodes.getClusterManagerNodeId(), parsedNodes.getClusterManagerNodeId()); + } + } + } + private DiscoveryNode buildDiscoveryNodeFromExisting(DiscoveryNode existing, Version newVersion) { return new DiscoveryNode( existing.getName(), From b9887b419698826aa28867bde7b4aeedd3ab78a2 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Tue, 7 May 2024 15:46:04 +0530 Subject: [PATCH 037/133] fixing checksum --- .../routingtable/IndexRoutingTableHeader.java | 17 +-------- .../IndexRoutingTableInputStream.java | 5 ++- .../IndexRoutingTableHeaderTests.java | 35 +++++++++++++++++++ 3 files changed, 38 insertions(+), 19 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 2a26f10ac4cf0..23ab700d5a34f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -74,10 +74,9 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); - final int version = in.readInt(); + final long version = in.readLong(); final int nodeVersion = in.readInt(); final String name = in.readString(); - verifyChecksum(in); assert version >= 0 : "Version must be non-negative [" + version + "]"; assert in.readByte() == -1 : "Header is not fully read"; return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); @@ -87,20 +86,6 @@ public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOExce } } - static void verifyChecksum(BufferedChecksumStreamInput in) throws IOException { - // This absolutely must come first, or else reading the checksum becomes part of the checksum - long expectedChecksum = in.getChecksum(); - long readChecksum = Integer.toUnsignedLong(in.readInt()); - if (readChecksum != expectedChecksum) { - throw new IOException( - "checksum verification failed - expected: 0x" - + Long.toHexString(expectedChecksum) - + ", got: 0x" - + Long.toHexString(readChecksum) - ); - } - } - static int readHeaderVersion(final StreamInput in) throws IOException { final int version; try { diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index 0b7f2cbb2326c..c6c777c70f10e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -71,10 +71,9 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long ve throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); this.bytesStreamOutput = new BytesStreamOutput(); this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); - logger.info("indexRoutingTable {}, version {}, nodeVersion {}", indexRoutingTable.prettyPrint(), version, nodeVersion); initialFill(indexRoutingTable.shards().size()); @@ -121,7 +120,7 @@ private void fill(byte[] buf) throws IOException { IndexShardRoutingTable.Builder.writeTo(next, out); //Add checksum for the file after all shards are done if(!shardIter.hasNext()) { - out.writeInt((int) out.getChecksum()); + out.writeLong(out.getChecksum()); } out.flush(); BytesReference bytesRef = bytesStreamOutput.bytes(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java new file mode 100644 index 0000000000000..068db554b4226 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.InputStreamDataInput; +import org.opensearch.Version; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { + + public void testWrite() throws IOException { + BytesStreamOutput out = new BytesStreamOutput(); + IndexRoutingTableHeader header = new IndexRoutingTableHeader(1, "dummyIndex", Version.V_3_0_0); + header.write(out); + + BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); + CodecUtil.checkHeader(new InputStreamDataInput(in),IndexRoutingTableHeader.INDEX_ROUTING_HEADER_CODEC, IndexRoutingTableHeader.INITIAL_VERSION, IndexRoutingTableHeader.CURRENT_VERSION ); + assertEquals(1, in.readLong()); + assertEquals(Version.V_3_0_0.id, in.readInt()); + assertEquals("dummyIndex", in.readString()); + } + +} From 15d06bc90143019700e30136396ff7cae3c25ccb Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 7 May 2024 16:58:43 +0530 Subject: [PATCH 038/133] test fix Signed-off-by: Shivansh Arora --- .../cluster/node/DiscoveryNodesTests.java | 55 +++++++++---------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java index 3097af64a439f..6a22601fefa93 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodesTests.java @@ -63,7 +63,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import java.util.stream.IntStream; import java.util.stream.StreamSupport; import static java.util.Collections.singletonMap; @@ -518,37 +517,33 @@ public void testToXContentInAPIMode() throws IOException { String expectedNodeAPUXContent = "%1$s\"node_%2$d\" : {\n" + "%1$s \"name\" : \"name_%2$d\",\n" + "%1$s \"ephemeral_id\" : \"%3$s\",\n" - + "%1$s \"transport_address\" : \"0.0.0.0:%4$d\",\n" + + "%1$s \"transport_address\" : \"%4$s\",\n" + "%1$s \"attributes\" : {%5$s}\n" + "%1$s}"; + logger.info(nodes); + verifyToXContentInContextMode( CONTEXT_MODE_API, nodes, - "{\n" - + " \"nodes\" : {\n" - + IntStream.range(0, nodes.getSize()) - .mapToObj( - i -> String.format( - Locale.ROOT, - expectedNodeAPUXContent, - " ", - i, - nodes.get("node_" + i).getEphemeralId(), - i + 1, - nodes.get("node_" + i).getAttributes().isEmpty() - ? " " - : "\n" + " \"custom\" : \"" + nodes.get("node_" + i).getAttributes().get("custom") + "\"\n " - ) - ) - .collect(Collectors.joining(",\n")) - + "\n" - + " }\n" - + "}" + "{\n" + " \"nodes\" : {\n" + nodes.getNodes().entrySet().stream().map(entry -> { + int id = Integer.parseInt(entry.getKey().split("_")[1]); + return String.format( + Locale.ROOT, + expectedNodeAPUXContent, + " ", + id, + entry.getValue().getEphemeralId(), + entry.getValue().getAddress().toString(), + entry.getValue().getAttributes().isEmpty() + ? " " + : "\n" + " \"custom\" : \"" + entry.getValue().getAttributes().get("custom") + "\"\n " + ); + }).collect(Collectors.joining(",\n")) + "\n" + " }\n" + "}" ); } - public void testXContentInGatewayMode() throws IOException { + public void testToXContentInGatewayMode() throws IOException { DiscoveryNodes nodes = buildDiscoveryNodes(); String expectedXContent = getExpectedXContentInGatewayMode(nodes); @@ -579,7 +574,7 @@ private String getExpectedXContentInGatewayMode(DiscoveryNodes nodes) { String expectedNodeAPUXContent = "%1$s\"node_%2$d\" : {\n" + "%1$s \"name\" : \"name_%2$d\",\n" + "%1$s \"ephemeral_id\" : \"%3$s\",\n" - + "%1$s \"transport_address\" : \"0.0.0.0:%4$d\",\n" + + "%1$s \"transport_address\" : \"%4$s\",\n" + "%1$s \"attributes\" : {%5$s},\n" + "%1$s \"host_name\" : \"0.0.0.0\",\n" + "%1$s \"host_address\" : \"0.0.0.0\",\n" @@ -587,20 +582,20 @@ private String getExpectedXContentInGatewayMode(DiscoveryNodes nodes) { + "%1$s \"roles\" : [%7$s]\n" + "%1$s}"; - return "{\n" + " \"nodes\" : {\n" + IntStream.range(0, nodes.getSize()).mapToObj(i -> { - String nodeId = "node_" + i; - DiscoveryNode node = nodes.get(nodeId); + return "{\n" + " \"nodes\" : {\n" + nodes.getNodes().entrySet().stream().map(entry -> { + int id = Integer.parseInt(entry.getKey().split("_")[1]); + DiscoveryNode node = entry.getValue(); String indent = " "; return String.format( Locale.ROOT, expectedNodeAPUXContent, indent, - i, + id, node.getEphemeralId(), - i + 1, + entry.getValue().getAddress().toString(), node.getAttributes().isEmpty() ? " " - : "\n" + indent + " \"custom\" : \"" + nodes.get("node_" + i).getAttributes().get("custom") + "\"\n " + indent, + : "\n" + indent + " \"custom\" : \"" + node.getAttributes().get("custom") + "\"\n " + indent, node.getVersion(), node.getRoles().isEmpty() ? " " From 87625d353fd73acf2886c1e5c83282ec6368eb24 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 9 May 2024 16:59:54 +0530 Subject: [PATCH 039/133] Add to and from XContent to ClusterBlock and ClusterBlocks Signed-off-by: Shivansh Arora --- .../cluster/block/ClusterBlock.java | 83 +++++++++++- .../cluster/block/ClusterBlockLevel.java | 8 ++ .../cluster/block/ClusterBlocks.java | 102 +++++++++++++- .../cluster/block/ClusterBlockTests.java | 76 ++++++++++- .../cluster/block/ClusterBlocksTests.java | 124 ++++++++++++++++++ 5 files changed, 385 insertions(+), 8 deletions(-) create mode 100644 server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java index 5fa897c0b1185..f048d1509ccdb 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java @@ -34,17 +34,22 @@ import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.util.set.Sets; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import java.io.IOException; import java.util.EnumSet; import java.util.Locale; import java.util.Objects; +import java.util.Set; /** * Blocks the cluster for concurrency @@ -54,6 +59,12 @@ @PublicApi(since = "1.0.0") public class ClusterBlock implements Writeable, ToXContentFragment { + static final String KEY_UUID = "uuid"; + static final String KEY_DESCRIPTION = "description"; + static final String KEY_RETRYABLE = "retryable"; + static final String KEY_DISABLE_STATE_PERSISTENCE = "disable_state_persistence"; + static final String KEY_LEVELS = "levels"; + private static final Set VALID_FIELDS = Sets.newHashSet(KEY_UUID, KEY_DESCRIPTION, KEY_RETRYABLE, KEY_DISABLE_STATE_PERSISTENCE, KEY_LEVELS); private final int id; @Nullable private final String uuid; @@ -156,14 +167,14 @@ public boolean disableStatePersistence() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Integer.toString(id)); if (uuid != null) { - builder.field("uuid", uuid); + builder.field(KEY_UUID, uuid); } - builder.field("description", description); - builder.field("retryable", retryable); + builder.field(KEY_DESCRIPTION, description); + builder.field(KEY_RETRYABLE, retryable); if (disableStatePersistence) { - builder.field("disable_state_persistence", disableStatePersistence); + builder.field(KEY_DISABLE_STATE_PERSISTENCE, disableStatePersistence); } - builder.startArray("levels"); + builder.startArray(KEY_LEVELS); for (ClusterBlockLevel level : levels) { builder.value(level.name().toLowerCase(Locale.ROOT)); } @@ -172,6 +183,68 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static ClusterBlock fromXContent(XContentParser parser, int id) throws IOException { + String uuid = null; + String description = null; + boolean retryable = false; + boolean disableStatePersistence = false; + EnumSet levels = EnumSet.noneOf(ClusterBlockLevel.class); + String currentFieldName = skipBlockID(parser); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + switch (Objects.requireNonNull(currentFieldName)) { + case KEY_UUID: + uuid = parser.text(); + break; + case KEY_DESCRIPTION: + description = parser.text(); + break; + case KEY_RETRYABLE: + retryable = parser.booleanValue(); + break; + case KEY_DISABLE_STATE_PERSISTENCE: + disableStatePersistence = parser.booleanValue(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (currentFieldName.equals(KEY_LEVELS)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + levels.add(ClusterBlockLevel.fromString(parser.text(), Locale.ROOT)); + } + } else { + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } else { + throw new IllegalArgumentException("unexpected token [" + token + "]"); + } + } + return new ClusterBlock(id, uuid, description, retryable, disableStatePersistence, false, null, levels); + } + + private static String skipBlockID(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + String currentFieldName = parser.currentName(); + if (VALID_FIELDS.contains(currentFieldName)) { + return currentFieldName; + } else { + // we have hit block id, just move on + parser.nextToken(); + } + } + } + return null; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(id); diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java index 5d3bf94aedb19..4940234c21ba6 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlockLevel.java @@ -35,6 +35,7 @@ import org.opensearch.common.annotation.PublicApi; import java.util.EnumSet; +import java.util.Locale; /** * What level to block the cluster @@ -51,4 +52,11 @@ public enum ClusterBlockLevel { public static final EnumSet ALL = EnumSet.allOf(ClusterBlockLevel.class); public static final EnumSet READ_WRITE = EnumSet.of(READ, WRITE); + + /* + * This method is used to convert a string to a ClusterBlockLevel. + * */ + public static ClusterBlockLevel fromString(String level, Locale locale) { + return ClusterBlockLevel.valueOf(level.toUpperCase(locale)); + } } diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index 304136166d515..e188374251d0d 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -42,6 +42,11 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.index.IndexModule; import java.io.IOException; @@ -63,7 +68,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class ClusterBlocks extends AbstractDiffable { +public class ClusterBlocks extends AbstractDiffable implements ToXContentFragment { public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(emptySet(), Map.of()); private final Set global; @@ -326,6 +331,16 @@ public static Diff readDiffFrom(StreamInput in) throws IOExceptio return AbstractDiffable.readDiffFrom(ClusterBlocks::readFrom, in); } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Builder.toXContext(this, builder, params); + return builder; + } + + public static ClusterBlocks fromXContent(XContentParser parser) throws IOException { + return Builder.fromXContent(parser); + } + /** * An immutable level holder. * @@ -427,10 +442,16 @@ public Builder removeGlobalBlock(int blockId) { } public Builder addIndexBlock(String index, ClusterBlock block) { + prepareIndexForBlocks(index); + indices.get(index).add(block); + return this; + } + + // initialize an index adding further blocks + private Builder prepareIndexForBlocks(String index) { if (!indices.containsKey(index)) { indices.put(index, new HashSet<>()); } - indices.get(index).add(block); return this; } @@ -480,5 +501,82 @@ public ClusterBlocks build() { } return new ClusterBlocks(unmodifiableSet(new HashSet<>(global)), indicesBuilder); } + + public static void toXContext(ClusterBlocks blocks, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject("blocks"); + if (blocks.global().isEmpty() == false) { + builder.startObject("global"); + for (ClusterBlock block : blocks.global()) { + block.toXContent(builder, params); + } + builder.endObject(); + } + + if (blocks.indices().isEmpty() == false) { + builder.startObject("indices"); + for (Map.Entry> entry : blocks.indices().entrySet()) { + builder.startObject(entry.getKey()); + for (ClusterBlock block : entry.getValue()) { + block.toXContent(builder, params); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + + public static ClusterBlocks fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + String currentFieldName = skipBlocksField(parser); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser); + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case "global": + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + builder.addGlobalBlock(ClusterBlock.fromXContent(parser, Integer.parseInt(currentFieldName))); + } + break; + case "indices": + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + String indexName = parser.currentName(); + parser.nextToken(); + // prepare for this index as we want to add this to ClusterBlocks even if there is no Block associated with it + builder.prepareIndexForBlocks(indexName); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + builder.addIndexBlock(indexName, ClusterBlock.fromXContent(parser, Integer.parseInt(currentFieldName))); + } + } + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + return builder.build(); + } + + private static String skipBlocksField(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + if ("blocks".equals(parser.currentName())) { + parser.nextToken(); + } else { + return parser.currentName(); + } + } + } + return null; + } } } diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java index 04e04bd96a7d3..ec6234168030a 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java @@ -35,13 +35,22 @@ import org.opensearch.Version; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.XContentTestUtils; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import static java.util.EnumSet.copyOf; @@ -136,7 +145,72 @@ public void testGetIndexBlockWithId() { assertThat(builder.build().getIndexBlockWithId("index", randomValueOtherThan(blockId, OpenSearchTestCase::randomInt)), nullValue()); } - private ClusterBlock randomClusterBlock() { + public void testToXContent() throws IOException { + ClusterBlock clusterBlock = randomClusterBlock(); + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + builder.startObject(); + clusterBlock.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + String expectedString = "{\n" + getExpectedXContentFragment(clusterBlock, " ") + "\n}"; + + assertEquals(expectedString, builder.toString()); + } + + public void testFromXContent() throws IOException { + doFromXContentTestWithRandomFields(false); + } + + public void testFromXContentWithRandomFields() throws IOException { + doFromXContentTestWithRandomFields(true); + } + + private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { + ClusterBlock clusterBlock = randomClusterBlock(); + boolean humanReadable = randomBoolean(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent(clusterBlock, mediaType, ToXContent.EMPTY_PARAMS, humanReadable); + + if (addRandomFields) { + String unsupportedField = "unsupported_field"; + BytesReference mutated = BytesReference.bytes( + XContentTestUtils.insertIntoXContent( + mediaType.xContent(), + originalBytes, + Collections.singletonList(Integer.toString(clusterBlock.id())), + () -> unsupportedField, + () -> randomAlphaOfLengthBetween(3, 10) + ) + ); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ClusterBlock.fromXContent(createParser(mediaType.xContent(), mutated), clusterBlock.id()) + ); + assertEquals(e.getMessage(), "unknown field [" + unsupportedField + "]"); + } else { + ClusterBlock parsedBlock = ClusterBlock.fromXContent(createParser(mediaType.xContent(), originalBytes), clusterBlock.id()); + assertEquals(clusterBlock, parsedBlock); + assertEquals(clusterBlock.description(), parsedBlock.description()); + assertEquals(clusterBlock.retryable(), parsedBlock.retryable()); + assertEquals(clusterBlock.disableStatePersistence(), parsedBlock.disableStatePersistence()); + assertArrayEquals(clusterBlock.levels().toArray(), parsedBlock.levels().toArray()); + } + } + + static String getExpectedXContentFragment(ClusterBlock clusterBlock, String indent) { + return indent + "\"" + clusterBlock.id() + "\" : {\n" + + (clusterBlock.uuid() != null ? + indent + " \"uuid\" : \""+ clusterBlock.uuid() + "\",\n" : "") + + indent + " \"description\" : \"" + clusterBlock.description() + "\",\n" + + indent + " \"retryable\" : " + clusterBlock.retryable() + ",\n" + + (clusterBlock.disableStatePersistence() ? + indent + " \"disable_state_persistence\" : " + clusterBlock.disableStatePersistence() + ",\n" : "") + + String.format(indent + " \"levels\" : [%s]\n", clusterBlock.levels().isEmpty() ? " " : + "\n" + String.join(",\n", clusterBlock.levels().stream().map(level -> indent + " \"" + level.name().toLowerCase(Locale.ROOT) + "\"").toArray(String[]::new)) + "\n " + indent) + + indent + "}"; + } + + static ClusterBlock randomClusterBlock() { final String uuid = randomBoolean() ? UUIDs.randomBase64UUID() : null; final List levels = Arrays.asList(ClusterBlockLevel.values()); return new ClusterBlock( diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java new file mode 100644 index 0000000000000..c7710c3f397b3 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java @@ -0,0 +1,124 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.block; + +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.XContentTestUtils; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.cluster.block.ClusterBlockTests.getExpectedXContentFragment; +import static org.opensearch.cluster.block.ClusterBlockTests.randomClusterBlock; + + +public class ClusterBlocksTests extends OpenSearchTestCase { + public void testToXContent() throws IOException { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + builder.startObject(); + clusterBlocks.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + String expectedXContent = "{\n" + + " \"blocks\" : {\n" + + String.format("%s", clusterBlocks.global().isEmpty() ? "" : + " \"global\" : {\n" + + clusterBlocks.global().stream().map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")).collect(Collectors.joining(",\n")) + + "\n }" + (!clusterBlocks.indices().isEmpty() ? "," : "") + "\n") + + String.format("%s", clusterBlocks.indices().isEmpty() ? "" : + " \"indices\" : {\n" + + clusterBlocks.indices().entrySet().stream().map(entry -> + " \"" + entry.getKey() + "\" : {" + (entry.getValue().isEmpty() ? " }" : "\n" + + entry.getValue().stream().map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")).collect(Collectors.joining(",\n")) + + "\n }") + ).collect(Collectors.joining(",\n")) + + "\n }\n") + + " }\n" + + "}"; + + assertEquals(expectedXContent, builder.toString()); + } + + public void testFromXContent() throws IOException { + doFromXContentTestWithRandomFields(false); + } + + public void testFromXContentWithRandomFields() throws IOException { + doFromXContentTestWithRandomFields(true); + } + + private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IOException { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + boolean humanReadable = randomBoolean(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent(clusterBlocks, mediaType, ToXContent.EMPTY_PARAMS, humanReadable); + + if (addRandomFields) { + String unsupportedField = "unsupported_field"; + BytesReference mutated = BytesReference.bytes( + XContentTestUtils.insertIntoXContent( + mediaType.xContent(), + originalBytes, + Collections.singletonList("blocks"), + () -> unsupportedField, + () -> randomAlphaOfLengthBetween(3, 10) + ) + ); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> ClusterBlocks.fromXContent(createParser(mediaType.xContent(), mutated)) + ); + assertEquals("unknown field ["+ unsupportedField +"]", exception.getMessage()); + } else { + try (XContentParser parser = createParser(JsonXContent.jsonXContent, originalBytes)) { + ClusterBlocks parsedClusterBlocks = ClusterBlocks.fromXContent(parser); + assertEquals(clusterBlocks.global().size(), parsedClusterBlocks.global().size()); + assertEquals(clusterBlocks.indices().size(), parsedClusterBlocks.indices().size()); + clusterBlocks.global().forEach(clusterBlock -> assertTrue(parsedClusterBlocks.global().contains(clusterBlock))); + clusterBlocks.indices().forEach((key, value) -> { + assertTrue(parsedClusterBlocks.indices().containsKey(key)); + value.forEach(clusterBlock -> assertTrue(parsedClusterBlocks.indices().get(key).contains(clusterBlock))); + }); + } + } + } + + private ClusterBlocks randomClusterBlocks() { + int randomGlobalBlocks = randomIntBetween(0, 10); + Set globalBlocks = new HashSet<>(); + for (int i = 0; i < randomGlobalBlocks; i++) { + globalBlocks.add(randomClusterBlock()); + } + + int randomIndices = randomIntBetween(0, 10); + Map> indexBlocks = new HashMap<>(); + for (int i = 0; i < randomIndices; i++) { + int randomIndexBlocks = randomIntBetween(0, 10); + Set blocks = new HashSet<>(); + for (int j = 0; j < randomIndexBlocks; j++) { + blocks.add(randomClusterBlock()); + } + indexBlocks.put("index-" + i, blocks); + } + return new ClusterBlocks(globalBlocks, indexBlocks); + } +} From 032ced2211edbcac7f643119b72a10e3a25188aa Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 10 May 2024 16:57:20 +0530 Subject: [PATCH 040/133] Address further PR comments Signed-off-by: Shivansh Arora --- .../cluster/metadata/IndexMetadata.java | 1 - .../remote/ClusterMetadataManifest.java | 3 +- .../remote/RemoteClusterStateService.java | 143 ++++++------------ .../RemoteClusterStateServiceTests.java | 49 ++++++ 4 files changed, 94 insertions(+), 102 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index ec2b69bbda247..80b78cfe154f1 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -753,7 +753,6 @@ public String getIndexUUID() { return index.getUUID(); } - /** * Test whether the current index UUID is the same as the given one. Returns true if either are _na_ */ diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 5c1510f96f0b9..bf02c73ca560b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -735,6 +735,7 @@ private static String uploadedFilename(Object[] fields) { PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD); } + static final String COMPONENT_PREFIX = "index--"; private final String indexName; private final String indexUUID; private final String uploadedFilename; @@ -757,7 +758,7 @@ public String getUploadedFilePath() { @Override public String getComponent() { - return getIndexName(); + return COMPONENT_PREFIX + getIndexName(); } public String getUploadedFilename() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index fa3d5f3773685..4ceb23126b0a5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -158,13 +158,13 @@ public class RemoteClusterStateService implements Closeable { new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); /** - * Manifest format compatible with older codec v1, where global metadata was missing. + * Manifest format compatible with older codec v1, where codec versions/global metadata was introduced. */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); /** - * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. + * Manifest format compatible with codec v2, where global metadata file is replaced with multiple metadata attribute files */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( "cluster-metadata-manifest", @@ -216,7 +216,7 @@ public class RemoteClusterStateService implements Closeable { + "updated : [{}], custom metadata updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; - public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; + public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. @@ -335,33 +335,15 @@ public ClusterMetadataManifest writeIncrementalMetadata( } assert previousClusterState.metadata().coordinationMetadata().term() == clusterState.metadata().coordinationMetadata().term(); - // Write Global Metadata - - final boolean updateCoordinationMetadata = Metadata.isCoordinationMetadataEqual( - previousClusterState.metadata(), - clusterState.metadata() - ) == false; - final boolean updateSettingsMetadata = Metadata.isSettingsMetadataEqual( - previousClusterState.metadata(), - clusterState.metadata() - ) == false; - final boolean updateTemplatesMetadata = Metadata.isTemplatesMetadataEqual( - previousClusterState.metadata(), - clusterState.metadata() - ) == false; - final Map previousStateCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); + final Map customsToBeDeletedFromRemote = new HashMap<>(previousManifest.getCustomMetadataMap()); final Map customsToUpload = getUpdatedCustoms(clusterState, previousClusterState); final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); for (final String custom : clusterState.metadata().customs().keySet()) { // remove all the customs which are present currently - previousStateCustomMap.remove(custom); + customsToBeDeletedFromRemote.remove(custom); } - // Write Index Metadata - final Map previousStateIndexMetadataByName = new HashMap<>(); - for (final IndexMetadata indexMetadata : previousClusterState.metadata().indices().values()) { - previousStateIndexMetadataByName.put(indexMetadata.getIndex().getName(), indexMetadata); - } + final Map indicesToBeDeletedFromRemote = new HashMap<>(previousClusterState.metadata().indices()); int numIndicesUpdated = 0; int numIndicesUnchanged = 0; @@ -374,7 +356,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( Map prevIndexMetadataByName = new HashMap<>(); for (final IndexMetadata indexMetadata : clusterState.metadata().indices().values()) { String indexName = indexMetadata.getIndex().getName(); - final IndexMetadata prevIndexMetadata = previousStateIndexMetadataByName.get(indexName); + final IndexMetadata prevIndexMetadata = indicesToBeDeletedFromRemote.get(indexName); Long previousVersion = prevIndexMetadata != null ? prevIndexMetadata.getVersion() : null; if (previousVersion == null || indexMetadata.getVersion() != previousVersion) { logger.debug( @@ -388,34 +370,31 @@ public ClusterMetadataManifest writeIncrementalMetadata( prevIndexMetadataByName.put(indexName, prevIndexMetadata); } else { numIndicesUnchanged++; + // index unchanged it shouldn't be deleted from remote + indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName()); } - previousStateIndexMetadataByName.remove(indexMetadata.getIndex().getName()); } UploadedMetadataResults uploadedMetadataResults; - boolean firstUpload = !previousManifest.hasMetadataAttributesFiles(); // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, // If file is empty and codec is 1 then write global metadata. - if (firstUpload) { - uploadedMetadataResults = writeMetadataInParallel( - clusterState, - toUpload, - prevIndexMetadataByName, - clusterState.metadata().customs(), - true, - true, - true - ); - } else { - uploadedMetadataResults = writeMetadataInParallel( - clusterState, - toUpload, - prevIndexMetadataByName, - customsToUpload, - updateCoordinationMetadata, - updateSettingsMetadata, - updateTemplatesMetadata - ); - } + boolean firstUploadForSplitGlobalMetadata = !previousManifest.hasMetadataAttributesFiles(); + boolean updateCoordinationMetadata = firstUploadForSplitGlobalMetadata + || Metadata.isCoordinationMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; + ; + boolean updateSettingsMetadata = firstUploadForSplitGlobalMetadata + || Metadata.isSettingsMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; + boolean updateTemplatesMetadata = firstUploadForSplitGlobalMetadata + || Metadata.isTemplatesMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; + + uploadedMetadataResults = writeMetadataInParallel( + clusterState, + toUpload, + prevIndexMetadataByName, + firstUploadForSplitGlobalMetadata ? clusterState.metadata().customs() : customsToUpload, + updateCoordinationMetadata, + updateSettingsMetadata, + updateTemplatesMetadata + ); // update the map if the metadata was uploaded uploadedMetadataResults.uploadedIndexMetadata.forEach( @@ -423,22 +402,19 @@ public ClusterMetadataManifest writeIncrementalMetadata( ); allUploadedCustomMap.putAll(uploadedMetadataResults.uploadedCustomMetadataMap); // remove the data for removed custom/indices - previousStateCustomMap.keySet().forEach(allUploadedCustomMap::remove); - previousStateIndexMetadataByName.keySet().forEach(allUploadedIndexMetadata::remove); + customsToBeDeletedFromRemote.keySet().forEach(allUploadedCustomMap::remove); + indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove); + final ClusterMetadataManifest manifest = uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), previousManifest.getPreviousClusterUUID(), - firstUpload || updateCoordinationMetadata - ? uploadedMetadataResults.uploadedCoordinationMetadata - : previousManifest.getCoordinationMetadata(), - firstUpload || updateSettingsMetadata - ? uploadedMetadataResults.uploadedSettingsMetadata - : previousManifest.getSettingsMetadata(), - firstUpload || updateTemplatesMetadata - ? uploadedMetadataResults.uploadedTemplatesMetadata - : previousManifest.getTemplatesMetadata(), - firstUpload || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), + updateCoordinationMetadata ? uploadedMetadataResults.uploadedCoordinationMetadata : previousManifest.getCoordinationMetadata(), + updateSettingsMetadata ? uploadedMetadataResults.uploadedSettingsMetadata : previousManifest.getSettingsMetadata(), + updateTemplatesMetadata ? uploadedMetadataResults.uploadedTemplatesMetadata : previousManifest.getTemplatesMetadata(), + firstUploadForSplitGlobalMetadata || !customsToUpload.isEmpty() + ? allUploadedCustomMap + : previousManifest.getCustomMetadataMap(), false ); deleteStaleClusterMetadata(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), RETAINED_MANIFESTS); @@ -597,23 +573,23 @@ private UploadedMetadataResults writeMetadataInParallel( } UploadedMetadataResults response = new UploadedMetadataResults(); results.forEach((name, uploadedMetadata) -> { - if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class)) { - response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); - } else if (uploadedMetadata.getComponent().contains(CUSTOM_METADATA)) { + if (name.contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; response.uploadedCustomMetadataMap.put( custom, new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename()) ); - } else if (COORDINATION_METADATA.equals(uploadedMetadata.getComponent())) { + } else if (COORDINATION_METADATA.equals(name)) { response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; - } else if (SETTING_METADATA.equals(uploadedMetadata.getComponent())) { + } else if (SETTING_METADATA.equals(name)) { response.uploadedSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; - } else if (TEMPLATES_METADATA.equals(uploadedMetadata.getComponent())) { + } else if (TEMPLATES_METADATA.equals(name)) { response.uploadedTemplatesMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else if (name.contains(UploadedIndexMetadata.COMPONENT_PREFIX)) { + response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); } else { - throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); + throw new IllegalStateException("Unknown metadata component name " + name); } }); return response; @@ -786,39 +762,6 @@ public void start() { blobStoreRepository = (BlobStoreRepository) repository; } - private ClusterMetadataManifest uploadV1Manifest( - ClusterState clusterState, - List uploadedIndexMetadata, - String previousClusterUUID, - String globalMetadataFileName, - boolean committed - ) throws IOException { - synchronized (this) { - final String manifestFileName = getManifestFileName( - clusterState.term(), - clusterState.version(), - committed, - ClusterMetadataManifest.CODEC_V1 - ); - ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() - .clusterTerm(clusterState.term()) - .stateVersion(clusterState.getVersion()) - .clusterUUID(clusterState.metadata().clusterUUID()) - .stateUUID(clusterState.stateUUID()) - .opensearchVersion(Version.CURRENT) - .nodeId(nodeId) - .committed(committed) - .codecVersion(ClusterMetadataManifest.CODEC_V1) - .globalMetadataFileName(globalMetadataFileName) - .indices(uploadedIndexMetadata) - .previousClusterUUID(previousClusterUUID) - .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) - .build(); - writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); - return manifest; - } - } - private ClusterMetadataManifest uploadManifest( ClusterState clusterState, List uploadedIndexMetadata, diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index eb119a82b36eb..9797955e6a1bf 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -803,6 +803,55 @@ public void testIndexMetadataOnlyUpdated() throws IOException { }); } + public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException { + // setup + mockBlobStoreObjects(); + + // Initial cluster state with index. + final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); + remoteClusterStateService.start(); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_"); + String initialIndex = "test-index"; + Index index1 = new Index("test-index-1", "index-uuid-1"); + Index index2 = new Index("test-index-2", "index-uuid-2"); + final Settings idxSettings1 = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index1.getUUID()) + .build(); + final IndexMetadata indexMetadata1 = new IndexMetadata.Builder(index1.getName()).settings(idxSettings1) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + final Settings idxSettings2 = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index2.getUUID()) + .build(); + final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(index2.getName()).settings(idxSettings2) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + ClusterState clusterState1 = ClusterState.builder(initialClusterState) + .metadata( + Metadata.builder(initialClusterState.getMetadata()) + .put(indexMetadata1, false) + .put(indexMetadata2, false) + .remove(initialIndex) + .build() + ) + .build(); + ClusterMetadataManifest manifest1 = remoteClusterStateService.writeIncrementalMetadata( + initialClusterState, + clusterState1, + initialManifest + ); + // verify that initial index is removed, and new index are added + assertTrue(initialManifest.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(initialIndex))); + assertFalse(manifest1.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(initialIndex))); + assertTrue(manifest1.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(index1.getName()))); + assertTrue(manifest1.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(index2.getName()))); + + } + private void verifyMetadataAttributeOnlyUpdated( Function clusterStateUpdater, BiConsumer assertions From e81db2105cdaa1ecfcbb6a60ec73cf96f34294a5 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 13 May 2024 14:38:11 +0530 Subject: [PATCH 041/133] Fix test failures Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateService.java | 4 +- .../RemoteClusterStateServiceTests.java | 43 +++++++++++++++---- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 4ceb23126b0a5..ac821cd15a5b3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -370,9 +370,9 @@ public ClusterMetadataManifest writeIncrementalMetadata( prevIndexMetadataByName.put(indexName, prevIndexMetadata); } else { numIndicesUnchanged++; - // index unchanged it shouldn't be deleted from remote - indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName()); } + // index present in current cluster state + indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName()); } UploadedMetadataResults uploadedMetadataResults; // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 9797955e6a1bf..1b242b921c0d7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -814,27 +814,27 @@ public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException { String initialIndex = "test-index"; Index index1 = new Index("test-index-1", "index-uuid-1"); Index index2 = new Index("test-index-2", "index-uuid-2"); - final Settings idxSettings1 = Settings.builder() + Settings idxSettings1 = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_INDEX_UUID, index1.getUUID()) .build(); - final IndexMetadata indexMetadata1 = new IndexMetadata.Builder(index1.getName()).settings(idxSettings1) + IndexMetadata indexMetadata1 = new IndexMetadata.Builder(index1.getName()).settings(idxSettings1) .numberOfShards(1) .numberOfReplicas(0) .build(); - final Settings idxSettings2 = Settings.builder() + Settings idxSettings2 = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_INDEX_UUID, index2.getUUID()) .build(); - final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(index2.getName()).settings(idxSettings2) + IndexMetadata indexMetadata2 = new IndexMetadata.Builder(index2.getName()).settings(idxSettings2) .numberOfShards(1) .numberOfReplicas(0) .build(); ClusterState clusterState1 = ClusterState.builder(initialClusterState) .metadata( Metadata.builder(initialClusterState.getMetadata()) - .put(indexMetadata1, false) - .put(indexMetadata2, false) + .put(indexMetadata1, true) + .put(indexMetadata2, true) .remove(initialIndex) .build() ) @@ -845,11 +845,36 @@ public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException { initialManifest ); // verify that initial index is removed, and new index are added + assertEquals(1, initialManifest.getIndices().size()); + assertEquals(2, manifest1.getIndices().size()); assertTrue(initialManifest.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(initialIndex))); assertFalse(manifest1.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(initialIndex))); - assertTrue(manifest1.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(index1.getName()))); - assertTrue(manifest1.getIndices().stream().anyMatch(indexMetadata -> indexMetadata.getIndexName().equals(index2.getName()))); - + // update index1, index2 is unchanged + indexMetadata1 = new IndexMetadata.Builder(indexMetadata1).version(indexMetadata1.getVersion() + 1).build(); + ClusterState clusterState2 = ClusterState.builder(clusterState1) + .metadata(Metadata.builder(clusterState1.getMetadata()).put(indexMetadata1, true).build()) + .build(); + ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1); + // index1 is updated + assertEquals(2, manifest2.getIndices().size()); + assertEquals( + 1, + manifest2.getIndices().stream().filter(uploadedIndex -> uploadedIndex.getIndexName().equals(index1.getName())).count() + ); + assertNotEquals( + manifest2.getIndices() + .stream() + .filter(uploadedIndex -> uploadedIndex.getIndexName().equals(index1.getName())) + .findFirst() + .get() + .getUploadedFilename(), + manifest1.getIndices() + .stream() + .filter(uploadedIndex -> uploadedIndex.getIndexName().equals(index1.getName())) + .findFirst() + .get() + .getUploadedFilename() + ); } private void verifyMetadataAttributeOnlyUpdated( From 04140ff03ca9d4764fc0421b3a3cd1e0811a8653 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 13 May 2024 14:45:39 +0530 Subject: [PATCH 042/133] Made minor requested changes Signed-off-by: Shivansh Arora --- .../gateway/remote/RemoteClusterStateCleanupManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 4f3f3b3bf3599..9ac70ae2a2bfe 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -68,7 +68,7 @@ public class RemoteClusterStateCleanupManager implements Closeable { private final RemoteClusterStateService remoteClusterStateService; private final RemotePersistenceStats remoteStateStats; private BlobStoreTransferService blobStoreTransferService; - private volatile TimeValue staleFileCleanupInterval; + private TimeValue staleFileCleanupInterval; private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private AsyncStaleFileDeletion staleFileDeletionTask; private long lastCleanupAttemptStateVersion; @@ -119,9 +119,9 @@ void cleanUpStaleFiles() { ClusterState currentAppliedState = clusterApplierService.state(); if (currentAppliedState.nodes().isLocalNodeElectedClusterManager()) { long cleanUpAttemptStateVersion = currentAppliedState.version(); - if (cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion > SKIP_CLEANUP_STATE_CHANGES - && Strings.isNotEmpty(currentAppliedState.getClusterName().value()) - && Strings.isNotEmpty(currentAppliedState.metadata().clusterUUID())) { + assert Strings.isNotEmpty(currentAppliedState.getClusterName().value()) : "cluster name is not set"; + assert Strings.isNotEmpty(currentAppliedState.metadata().clusterUUID()) : "cluster uuid is not set"; + if (cleanUpAttemptStateVersion - lastCleanupAttemptStateVersion > SKIP_CLEANUP_STATE_CHANGES) { logger.info( "Cleaning up stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates", currentAppliedState.getClusterName().value(), From c20c6ac72b449c84e6e54acf4092ecab1806dc17 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Wed, 8 May 2024 11:09:54 +0530 Subject: [PATCH 043/133] Add read flow for IndexRoutingTable Signed-off-by: Arpit Bandejiya --- .../routingtable/IndexRoutingTableHeader.java | 10 +-- .../IndexRoutingTableInputStreamReader.java | 77 +++++++++++++++++++ .../IndexRoutingTableInputStreamTests.java | 61 +++++++++++++++ 3 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index 23ab700d5a34f..e29ce5a79dc02 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -65,22 +65,18 @@ public void write(StreamOutput out) throws IOException { /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} - * @param inBytes - * @param source - * @return + * @param in + * @return IndexRoutingTableHeader * @throws IOException */ - public IndexRoutingTableHeader read(byte[] inBytes, String source) throws IOException { + public static IndexRoutingTableHeader read(BufferedChecksumStreamInput in) throws IOException { try { - try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(new BytesStreamInput(inBytes), source)) { readHeaderVersion(in); final long version = in.readLong(); final int nodeVersion = in.readInt(); final String name = in.readString(); assert version >= 0 : "Version must be non-negative [" + version + "]"; - assert in.readByte() == -1 : "Header is not fully read"; return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); - } } catch (EOFException e) { throw new IOException("index routing header truncated", e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java new file mode 100644 index 0000000000000..35ae9f287d7f2 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.routingtable; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.IndexShardRoutingTable; +import org.opensearch.common.io.stream.BufferedChecksumStreamInput; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IndexRoutingTableInputStreamReader { + + private final StreamInput streamInput; + + private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); + + public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { + this.streamInput = new InputStreamStreamInput(inputStream); + } + + public Map read() throws IOException { + try { + try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { + // Read the Table Header first + IndexRoutingTableHeader.read(in); + int shards = in.readVInt(); + logger.info("Number of Index Routing Table {}", shards); + Map indicesRouting = new HashMap(Collections.EMPTY_MAP); + for(int i=0; i { + try { + logger.info("IndexShardRoutingTables: {}", indexShardRoutingTables); + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables, + initialRoutingTable.version(), Version.CURRENT); + + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + Map indexShardRoutingTableMap = reader.read(); + + logger.info("indexShardRoutingTableMap: {}", indexShardRoutingTableMap); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + +} From 381630ba109f3b16a32c27f541ad27f16e7a98e7 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Mon, 13 May 2024 18:08:24 +0530 Subject: [PATCH 044/133] Moving routing table version from IndexRouting stream to manifest Signed-off-by: Himshikha Gupta --- .../remote/ClusterMetadataManifest.java | 35 ++++++++++- .../routingtable/IndexRoutingTableHeader.java | 59 +++++-------------- .../IndexRoutingTableInputStream.java | 11 ++-- .../remote/ClusterMetadataManifestTests.java | 29 +++++++++ .../IndexRoutingTableHeaderTests.java | 14 ++--- .../IndexRoutingTableInputStreamTests.java | 12 ++-- 6 files changed, 92 insertions(+), 68 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 3222e44736bd4..c6b28214e47bc 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -49,6 +49,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField ROUTING_TABLE_VERSION_FIELD = new ParseField("routing_table_version"); private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); private static long term(Object[] fields) { @@ -99,8 +100,12 @@ private static String globalMetadataFileName(Object[] fields) { return (String) fields[11]; } + private static long routingTableVersion(Object[] fields) { + return (long) fields[12]; + } + private static List indicesRouting(Object[] fields) { - return (List) fields[12]; + return (List) fields[13]; } private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( @@ -154,6 +159,7 @@ private static List indicesRouting(Object[] fields) { indices(fields), previousClusterUUID(fields), clusterUUIDCommitted(fields), + routingTableVersion(fields), indicesRouting(fields) ) ); @@ -187,6 +193,7 @@ private static void declareParser(ConstructingObjectParser= CODEC_V2) { + parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); parser.declareObjectArray( ConstructingObjectParser.constructorArg(), (p, c) -> UploadedIndexMetadata.fromXContent(p), @@ -207,6 +214,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; public List getIndices() { @@ -257,6 +265,10 @@ public String getGlobalMetadataFileName() { return globalMetadataFileName; } + public long getRoutingTableVersion() { + return routingTableVersion; + } + public List getIndicesRouting() { return indicesRouting; } @@ -277,7 +289,7 @@ public ClusterMetadataManifest( boolean clusterUUIDCommitted ) { this(clusterTerm, version, clusterUUID, stateUUID, opensearchVersion, nodeId, committed, codecVersion, - globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, new ArrayList<>()); + globalMetadataFileName, indices, previousClusterUUID, clusterUUIDCommitted, -1, new ArrayList<>()); } public ClusterMetadataManifest( @@ -293,6 +305,7 @@ public ClusterMetadataManifest( List indices, String previousClusterUUID, boolean clusterUUIDCommitted, + long routingTableVersion, List indicesRouting ) { this.clusterTerm = clusterTerm; @@ -307,6 +320,7 @@ public ClusterMetadataManifest( this.indices = Collections.unmodifiableList(indices); this.previousClusterUUID = previousClusterUUID; this.clusterUUIDCommitted = clusterUUIDCommitted; + this.routingTableVersion = routingTableVersion; this.indicesRouting = Collections.unmodifiableList(indicesRouting); } @@ -325,14 +339,17 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_14_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); + this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); - this.indicesRouting =null; + this.routingTableVersion = -1; + this.indicesRouting = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; + this.routingTableVersion = -1; this.indicesRouting = null; } } @@ -371,6 +388,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } if (onOrAfterCodecVersion(CODEC_V2)) { + builder.field(ROUTING_TABLE_VERSION_FIELD.getPreferredName(), getRoutingTableVersion()); builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { for (UploadedIndexMetadata uploadedIndexMetadata : indicesRouting) { @@ -401,6 +419,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(globalMetadataFileName); } if (out.getVersion().onOrAfter(Version.V_2_14_0)) { + out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); } } @@ -426,6 +445,7 @@ public boolean equals(Object o) { && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(routingTableVersion, that.routingTableVersion) && Objects.equals(indicesRouting, that.indicesRouting); } @@ -444,6 +464,7 @@ public int hashCode() { committed, previousClusterUUID, clusterUUIDCommitted, + routingTableVersion, indicesRouting ); } @@ -484,6 +505,7 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; + private long routingTableVersion; private List indicesRouting; public Builder indices(List indices) { @@ -491,6 +513,11 @@ public Builder indices(List indices) { return this; } + public Builder routingTableVersion(long routingTableVersion) { + this.routingTableVersion = routingTableVersion; + return this; + } + public Builder indicesRouting(List indicesRouting) { this.indicesRouting = indicesRouting; return this; @@ -576,6 +603,7 @@ public Builder(ClusterMetadataManifest manifest) { this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; + this.routingTableVersion = manifest.routingTableVersion; this.indicesRouting = new ArrayList<>(manifest.indicesRouting); } @@ -593,6 +621,7 @@ public ClusterMetadataManifest build() { indices, previousClusterUUID, clusterUUIDCommitted, + routingTableVersion, indicesRouting ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java index e29ce5a79dc02..bc99fea9b5c09 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeader.java @@ -14,12 +14,6 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; -import org.opensearch.Version; -import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -31,52 +25,27 @@ */ public class IndexRoutingTableHeader { - private final long routingTableVersion; - - private final String indexName; - - private final Version nodeVersion; - public static final String INDEX_ROUTING_HEADER_CODEC = "index_routing_header_codec"; - public static final int INITIAL_VERSION = 1; - public static final int CURRENT_VERSION = INITIAL_VERSION; + private final String indexName; - public IndexRoutingTableHeader(long routingTableVersion, String indexName, Version nodeVersion) { - this.routingTableVersion = routingTableVersion; + public IndexRoutingTableHeader(String indexName) { this.indexName = indexName; - this.nodeVersion = nodeVersion; - } - - /** - * Returns the bytes reference for the {@link IndexRoutingTableHeader} - * @throws IOException - */ - public void write(StreamOutput out) throws IOException { - CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); - // Write version - out.writeLong(routingTableVersion); - out.writeInt(nodeVersion.id); - out.writeString(indexName); - - out.flush(); } /** * Reads the contents on the byte array into the corresponding {@link IndexRoutingTableHeader} + * * @param in * @return IndexRoutingTableHeader * @throws IOException */ - public static IndexRoutingTableHeader read(BufferedChecksumStreamInput in) throws IOException { + public static IndexRoutingTableHeader read(StreamInput in) throws IOException { try { - readHeaderVersion(in); - final long version = in.readLong(); - final int nodeVersion = in.readInt(); - final String name = in.readString(); - assert version >= 0 : "Version must be non-negative [" + version + "]"; - return new IndexRoutingTableHeader(version, name, Version.fromId(nodeVersion)); + readHeaderVersion(in); + final String name = in.readString(); + return new IndexRoutingTableHeader(name); } catch (EOFException e) { throw new IOException("index routing header truncated", e); } @@ -92,15 +61,19 @@ static int readHeaderVersion(final StreamInput in) throws IOException { return version; } - public long getRoutingTableVersion() { - return routingTableVersion; + /** + * Returns the bytes reference for the {@link IndexRoutingTableHeader} + * + * @throws IOException + */ + public void write(StreamOutput out) throws IOException { + CodecUtil.writeHeader(new OutputStreamDataOutput(out), INDEX_ROUTING_HEADER_CODEC, CURRENT_VERSION); + out.writeString(indexName); + out.flush(); } public String getIndexName() { return indexName; } - public Version getNodeVersion() { - return nodeVersion; - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index c6c777c70f10e..3c249894a2ace 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.Version; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; @@ -63,18 +62,18 @@ public class IndexRoutingTableInputStream extends InputStream { private final BytesStreamOutput bytesStreamOutput; private final BufferedChecksumStreamOutput out; - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion) throws IOException { - this(indexRoutingTable, version, nodeVersion, BUFFER_SIZE); + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws IOException { + this(indexRoutingTable, BUFFER_SIZE); } - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, long version, Version nodeVersion, int size) + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); - this.indexRoutingTableHeader = new IndexRoutingTableHeader(version, indexRoutingTable.getIndex().getName(), nodeVersion); + this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); this.bytesStreamOutput = new BytesStreamOutput(); this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); - logger.info("indexRoutingTable {}, version {}, nodeVersion {}", indexRoutingTable.prettyPrint(), version, nodeVersion); + logger.info("indexRoutingTable {}", indexRoutingTable.prettyPrint()); initialFill(indexRoutingTable.shards().size()); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 6c9a3201656d7..73933141ddf23 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -233,6 +233,35 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { } } + public void testClusterMetadataManifestXContentV2() throws IOException { + UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); + ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( + 1L, + 1L, + "test-cluster-uuid", + "test-state-uuid", + Version.CURRENT, + "test-node-id", + false, + ClusterMetadataManifest.CODEC_V2, + "test-metadata", + Collections.singletonList(uploadedIndexMetadata), + "prev-cluster-uuid", + true, + 1L, + Collections.singletonList(uploadedIndexMetadata) + ); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); + assertEquals(originalManifest, fromXContentManifest); + } + } + private List randomUploadedIndexMetadataList() { final int size = randomIntBetween(1, 10); final List uploadedIndexMetadataList = new ArrayList<>(size); diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java index 068db554b4226..0f42508615a26 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableHeaderTests.java @@ -8,28 +8,22 @@ package org.opensearch.gateway.remote.routingtable; -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.InputStreamDataInput; -import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.BytesStreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; public class IndexRoutingTableHeaderTests extends OpenSearchTestCase { - public void testWrite() throws IOException { + public void testIndexRoutingTableHeader() throws IOException { + IndexRoutingTableHeader header = new IndexRoutingTableHeader("dummyIndex"); BytesStreamOutput out = new BytesStreamOutput(); - IndexRoutingTableHeader header = new IndexRoutingTableHeader(1, "dummyIndex", Version.V_3_0_0); header.write(out); BytesStreamInput in = new BytesStreamInput(out.bytes().toBytesRef().bytes); - CodecUtil.checkHeader(new InputStreamDataInput(in),IndexRoutingTableHeader.INDEX_ROUTING_HEADER_CODEC, IndexRoutingTableHeader.INITIAL_VERSION, IndexRoutingTableHeader.CURRENT_VERSION ); - assertEquals(1, in.readLong()); - assertEquals(Version.V_3_0_0.id, in.readInt()); - assertEquals("dummyIndex", in.readString()); + IndexRoutingTableHeader headerRead = IndexRoutingTableHeader.read(in); + assertEquals("dummyIndex", headerRead.getIndexName()); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java index 3957d4602201d..b10a60943798a 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamTests.java @@ -33,9 +33,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; -public class IndexRoutingTableInputStreamTests extends ReplicationTrackerTestCase { +public class IndexRoutingTableInputStreamTests extends OpenSearchTestCase { - public void testRoutingTableInputStream() throws IOException { + public void testRoutingTableInputStream(){ Metadata metadata = Metadata.builder() .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) .build(); @@ -44,14 +44,14 @@ public void testRoutingTableInputStream() throws IOException { initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { try { - logger.info("IndexShardRoutingTables: {}", indexShardRoutingTables); - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables, - initialRoutingTable.version(), Version.CURRENT); + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); Map indexShardRoutingTableMap = reader.read(); - logger.info("indexShardRoutingTableMap: {}", indexShardRoutingTableMap); + assertEquals(1, indexShardRoutingTableMap.size()); + assertNotNull(indexShardRoutingTableMap.get("test")); + assertEquals(2,indexShardRoutingTableMap.get("test").shards().size()); } catch (IOException e) { throw new RuntimeException(e); } From a10b062f72de783ef2e5a1e7d6ecb25c41ba1f1a Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 14 May 2024 14:11:53 +0530 Subject: [PATCH 045/133] Draft changes for Diff manifest upload Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 45 ++-- .../org/opensearch/cluster/ClusterState.java | 30 +-- .../cluster/RepositoryCleanupInProgress.java | 36 +++ .../opensearch/cluster/RestoreInProgress.java | 211 +++++++++++++++--- .../cluster/SnapshotDeletionsInProgress.java | 51 +++++ .../cluster/node/DiscoveryNodes.java | 5 +- .../remote/ClusterMetadataManifest.java | 176 ++++++++++++++- .../RemoteClusterStateAttributesManager.java | 95 ++++++++ .../remote/RemoteClusterStateService.java | 79 ++++++- .../remote/RemoteClusterStateUtils.java | 10 +- .../gateway/remote/RemoteManifestManager.java | 8 +- .../remote/ClusterMetadataManifestTests.java | 10 +- .../RemoteClusterStateServiceTests.java | 1 - 13 files changed, 660 insertions(+), 97 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index d556206b8d341..91bba40375b49 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -77,31 +77,26 @@ public void testRemoteCleanupTaskUpdated() { RemoteClusterStateService.class ); - assertEquals( - 5, - remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes() - ); + assertEquals(5, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); // now disable - client().admin().cluster().prepareUpdateSettings() + client().admin() + .cluster() + .prepareUpdateSettings() .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), -1)) .get(); - assertEquals( - -1, - remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMillis() - ); + assertEquals(-1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMillis()); assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); // now set Clean up interval to 1 min - client().admin().cluster().prepareUpdateSettings() + client().admin() + .cluster() + .prepareUpdateSettings() .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) .get(); - assertEquals( - 1, - remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes() - ); + assertEquals(1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); } public void testRemoteCleanupOnlyAfter10Updates() throws Exception { @@ -116,7 +111,9 @@ public void testRemoteCleanupOnlyAfter10Updates() throws Exception { ); // set cleanup interval to 1 min - client().admin().cluster().prepareUpdateSettings() + client().admin() + .cluster() + .prepareUpdateSettings() .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) .get(); @@ -136,20 +133,29 @@ public void testRemoteCleanupOnlyAfter10Updates() throws Exception { BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); assertBusy(() -> { - assertEquals(RETAINED_MANIFESTS - 1, repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size()); + assertEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); }, 1, TimeUnit.MINUTES); replicaCount = updateReplicaCountNTimes(8, replicaCount); // wait for 1 min, to ensure that clean up task ran and didn't clean up stale files because it was less than 10 Thread.sleep(60000); - assertNotEquals(RETAINED_MANIFESTS - 1, repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size()); + assertNotEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); // Do 2 more updates, now since the total successful state changes are more than 10, stale files will be cleaned up replicaCount = updateReplicaCountNTimes(2, replicaCount); assertBusy(() -> { - assertEquals(RETAINED_MANIFESTS - 1, repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size()); + assertEquals( + RETAINED_MANIFESTS - 1, + repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() + ); }, 1, TimeUnit.MINUTES); Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( @@ -299,7 +305,8 @@ private void setReplicaCount(int replicaCount) { } private int updateReplicaCountNTimes(int n, int initialCount) { - int newReplicaCount = randomIntBetween(0, 3);; + int newReplicaCount = randomIntBetween(0, 3); + ; for (int i = 0; i < n; i++) { while (newReplicaCount == initialCount) { newReplicaCount = randomIntBetween(0, 3); diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index 9e63f961d241d..e38936d3e2622 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -496,38 +496,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (metrics.contains(Metric.BLOCKS)) { - builder.startObject("blocks"); - - if (blocks().global().isEmpty() == false) { - builder.startObject("global"); - for (ClusterBlock block : blocks().global()) { - block.toXContent(builder, params); - } - builder.endObject(); - } - - if (blocks().indices().isEmpty() == false) { - builder.startObject("indices"); - for (final Map.Entry> entry : blocks().indices().entrySet()) { - builder.startObject(entry.getKey()); - for (ClusterBlock block : entry.getValue()) { - block.toXContent(builder, params); - } - builder.endObject(); - } - builder.endObject(); - } - - builder.endObject(); + blocks().toXContent(builder, params); } // nodes if (metrics.contains(Metric.NODES)) { - builder.startObject("nodes"); - for (DiscoveryNode node : nodes) { - node.toXContent(builder, params); - } - builder.endObject(); + nodes.toXContent(builder, params); } // meta data diff --git a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java index 72a3519aca6f8..636db24d1dbad 100644 --- a/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RepositoryCleanupInProgress.java @@ -33,12 +33,15 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.repositories.RepositoryOperation; import java.io.IOException; @@ -101,6 +104,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); { builder.field("repository", entry.repository); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field("repository_state_id", entry.repositoryStateId); + } // else we don't serialize it } builder.endObject(); } @@ -108,6 +114,36 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static RepositoryCleanupInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + XContentParserUtils.ensureFieldName(parser, parser.currentToken(), TYPE); + parser.nextToken(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List entries = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + String repository = null; + long repositoryStateId = -1L; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + parser.nextToken(); + if ("repository".equals(currentFieldName)) { + repository = parser.text(); + } else if ("repository_state_id".equals(currentFieldName)) { + // only XContent parsed with {@link Metadata.CONTEXT_MODE_GATEWAY} will have the repository state id and can be deserialized + repositoryStateId = parser.longValue(); + } else { + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + entries.add(new Entry(repository, repositoryStateId)); + } + return new RepositoryCleanupInProgress(entries); + } + @Override public String toString() { return Strings.toString(MediaTypeRegistry.JSON, this); diff --git a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java index 769f97373f7b7..627a04bd8277b 100644 --- a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java @@ -32,18 +32,25 @@ package org.opensearch.cluster; +import org.elasticsearch.snapshots.SnapshotId; import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.snapshots.Snapshot; +import org.opensearch.snapshots.SnapshotId; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -52,6 +59,8 @@ import java.util.Objects; import java.util.UUID; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * Meta data about restore processes that are currently executing * @@ -144,7 +153,7 @@ public RestoreInProgress build() { * @opensearch.api */ @PublicApi(since = "1.0.0") - public static class Entry { + public static class Entry implements ToXContentFragment { private final String uuid; private final State state; private final Snapshot snapshot; @@ -236,6 +245,135 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(uuid, snapshot, state, indices, shards); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + boolean isGatewayXContent = params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API) + .equals(Metadata.CONTEXT_MODE_GATEWAY); + builder.startObject(); + builder.field("snapshot", snapshot().getSnapshotId().getName()); + builder.field("repository", snapshot().getRepository()); + builder.field("state", state()); + if (isGatewayXContent) { + builder.field("snapshot_uuid", snapshot().getSnapshotId().getUUID()); + builder.field("uuid", uuid()); + } + builder.startArray("indices"); + { + for (String index : indices()) { + builder.value(index); + } + } + builder.endArray(); + builder.startArray("shards"); + { + for (final Map.Entry shardEntry : shards.entrySet()) { + ShardId shardId = shardEntry.getKey(); + ShardRestoreStatus status = shardEntry.getValue(); + builder.startObject(); + { + builder.field("index", shardId.getIndex()); + builder.field("shard", shardId.getId()); + builder.field("state", status.state()); + if (isGatewayXContent) { + builder.field("index_uuid", shardId.getIndex().getUUID()); + if (status.nodeId() != null) builder.field("node_id", status.nodeId()); + if (status.reason() != null) builder.field("reason", status.reason()); + } + } + builder.endObject(); + } + } + + builder.endArray(); + builder.endObject(); + return builder; + } + + public static Entry fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + String snapshotName = null; + String snapshotRepository = null; + String snapshotUUID = null; + int state = -1; + String uuid = null; + List indices = new ArrayList<>(); + Map shards = new HashMap<>(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case "snapshot": + snapshotName = parser.text(); + break; + case "repository": + snapshotRepository = parser.text(); + break; + case "state": + state = parser.intValue(); + break; + case "snapshot_uuid": + snapshotUUID = parser.text(); + break; + case "uuid": + uuid = parser.text(); + break; + case "indices": + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + indices.add(parser.text()); + } + break; + case "shards": + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + String indexName = null; + String indexUUID = null; + int shardId = -1; + int restoreState = -1; + String nodeId = null; + String reason = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentShardFieldName = parser.currentName(); + parser.nextToken(); + switch (currentShardFieldName) { + case "index": + indexName = parser.text(); + break; + case "shard": + shardId = parser.intValue(); + break; + case "state": + restoreState = parser.intValue(); + break; + case "index_uuid": + indexUUID = parser.text(); + break; + case "node_id": + nodeId = parser.text(); + break; + case "reason": + reason = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + shards.put(new ShardId(indexName, indexUUID, shardId), new ShardRestoreStatus(nodeId, State.fromValue((byte) restoreState), reason)); + } + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + return new Entry(uuid, new Snapshot(snapshotRepository, new SnapshotId(snapshotName, snapshotUUID)), State.fromValue((byte) state), indices, shards); + } } /** @@ -495,46 +633,57 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startArray("snapshots"); for (final Entry entry : entries.values()) { - toXContent(entry, builder); + toXContent(entry, builder, ToXContent.EMPTY_PARAMS); } builder.endArray(); return builder; } + public static RestoreInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + final Map entries = new HashMap<>(); + XContentParserUtils.ensureFieldName(parser, parser.currentToken(), "snapshots"); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + final Entry entry = Entry.fromXContent(parser); + entries.put(entry.uuid, entry); + } + return new RestoreInProgress(entries); + } + /** * Serializes single restore operation * * @param entry restore operation metadata * @param builder XContent builder + * @param params */ - public void toXContent(Entry entry, XContentBuilder builder) throws IOException { - builder.startObject(); - builder.field("snapshot", entry.snapshot().getSnapshotId().getName()); - builder.field("repository", entry.snapshot().getRepository()); - builder.field("state", entry.state()); - builder.startArray("indices"); - { - for (String index : entry.indices()) { - builder.value(index); - } - } - builder.endArray(); - builder.startArray("shards"); - { - for (final Map.Entry shardEntry : entry.shards.entrySet()) { - ShardId shardId = shardEntry.getKey(); - ShardRestoreStatus status = shardEntry.getValue(); - builder.startObject(); - { - builder.field("index", shardId.getIndex()); - builder.field("shard", shardId.getId()); - builder.field("state", status.state()); - } - builder.endObject(); - } - } - - builder.endArray(); - builder.endObject(); + public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException { + entry.toXContent(builder, params); } } + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index e33245e02f75c..6ef755ee34ebb 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -40,6 +40,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.repositories.RepositoryOperation; import org.opensearch.snapshots.SnapshotId; @@ -52,6 +54,9 @@ import java.util.Objects; import java.util.Set; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; + /** * A class that represents the snapshot deletions that are in progress in the cluster. * @@ -202,6 +207,52 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureFieldName(parser, parser.currentToken(), TYPE); + parser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + List entries = new ArrayList<>(); + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + String repository, repositoryStateId; + List snapshotNames = new ArrayList<>(); + TimeValue startTime; + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case "repository": + repository = parser.text(); + break; + case "snapshots": + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + snapshotNames.add(parser.text()); + } + break; + case "start_time_millis": + startTime = TimeValue.timeValueMillis(parser.longValue()); + break; + case "start_time": + startTime = TimeValue.parseTimeValue(parser.text(), "start_time"); + break; + case "repository_state_id": + repositoryStateId = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + fieldName + "]"); + } + } +// entries.add(new Entry()) + } + return SnapshotDeletionsInProgress.of(entries); + } + + @Override public String toString() { StringBuilder builder = new StringBuilder("SnapshotDeletionsInProgress["); diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java index d281021624fd0..c5ecc2a188079 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNodes.java @@ -48,6 +48,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import java.io.IOException; import java.util.ArrayList; @@ -595,9 +596,7 @@ public static DiscoveryNodes fromXContent(XContentParser parser) throws IOExcept if (parser.currentToken() == XContentParser.Token.START_OBJECT) { parser.nextToken(); } - if (parser.currentToken() != XContentParser.Token.FIELD_NAME) { - throw new IllegalArgumentException("expected field name but got a " + parser.currentToken()); - } + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); XContentParser.Token token; String currentFieldName = parser.currentName(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 023eb06b8eb86..dccb11f25e5ef 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -9,6 +9,9 @@ package org.opensearch.gateway.remote; import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; @@ -18,6 +21,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -41,6 +45,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. + public static final int CODEC_V3 = 3; // In Codec V3, we introduced diff as part of manifest. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -58,6 +63,9 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField UPLOADED_SETTINGS_METADATA = new ParseField("uploaded_settings_metadata"); private static final ParseField UPLOADED_TEMPLATES_METADATA = new ParseField("uploaded_templates_metadata"); private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata"); + private static final ParseField UPLOADED_DISCOVERY_NODES_METADATA = new ParseField("uploaded_discovery_nodes_metadata"); + private static final ParseField UPLOADED_CLUSTER_BLOCKS_METADATA = new ParseField("uploaded_cluster_blocks_metadata"); + private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest"); private static long term(Object[] fields) { return (long) fields[0]; @@ -237,6 +245,8 @@ private static void declareParser(ConstructingObjectParser uploadedCustomMetadataMap; private final List indices; private final long clusterTerm; @@ -248,6 +258,7 @@ private static void declareParser(ConstructingObjectParser getIndices() { return indices; @@ -309,6 +320,18 @@ public UploadedMetadataAttribute getTemplatesMetadata() { return uploadedTemplatesMetadata; } + public UploadedMetadataAttribute getDiscoveryNodesMetadata() { + return uploadedDiscoveryNodesMetadata; + } + + public UploadedMetadataAttribute getClusterBlocksMetadata() { + return uploadedClusterBlocksMetadata; + } + + public ClusterDiffManifest getDiffManifest() { + return diffManifest; + } + public Map getCustomMetadataMap() { return uploadedCustomMetadataMap; } @@ -336,7 +359,10 @@ public ClusterMetadataManifest( UploadedMetadataAttribute uploadedCoordinationMetadata, UploadedMetadataAttribute uploadedSettingsMetadata, UploadedMetadataAttribute uploadedTemplatesMetadata, - Map uploadedCustomMetadataMap + Map uploadedCustomMetadataMap, + UploadedMetadataAttribute discoveryNodesMetadata, + UploadedMetadataAttribute clusterBlocksMetadata, + ClusterDiffManifest diffManifest ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -356,6 +382,9 @@ public ClusterMetadataManifest( this.uploadedCustomMetadataMap = Collections.unmodifiableMap( uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>() ); + this.uploadedDiscoveryNodesMetadata = discoveryNodesMetadata; + this.uploadedClusterBlocksMetadata = clusterBlocksMetadata; + this.diffManifest = diffManifest; } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -378,6 +407,10 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { in.readMap(StreamInput::readString, UploadedMetadataAttribute::new) ); this.globalMetadataFileName = null; + // ToDo: change the version check and initialize these + this.uploadedDiscoveryNodesMetadata = null; + this.uploadedClusterBlocksMetadata = null; + this.diffManifest = null; } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); @@ -385,6 +418,9 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedSettingsMetadata = null; this.uploadedTemplatesMetadata = null; this.uploadedCustomMetadataMap = null; + this.uploadedDiscoveryNodesMetadata = null; + this.uploadedClusterBlocksMetadata = null; + this.diffManifest = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; @@ -392,6 +428,9 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedSettingsMetadata = null; this.uploadedTemplatesMetadata = null; this.uploadedCustomMetadataMap = null; + this.uploadedDiscoveryNodesMetadata = null; + this.uploadedClusterBlocksMetadata = null; + this.diffManifest = null; } } @@ -438,11 +477,27 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws getTemplatesMetadata().toXContent(builder, params); builder.endObject(); } + if (getDiscoveryNodesMetadata() != null) { + builder.startObject(UPLOADED_DISCOVERY_NODES_METADATA.getPreferredName()); + getDiscoveryNodesMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getClusterBlocksMetadata() != null) { + builder.startObject(UPLOADED_CLUSTER_BLOCKS_METADATA.getPreferredName()); + getClusterBlocksMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getDiffManifest() != null) { + builder.startObject(DIFF_MANIFEST.getPreferredName()); + getDiffManifest().toXContent(builder, params); + builder.endObject(); + } builder.startObject(UPLOADED_CUSTOM_METADATA.getPreferredName()); for (UploadedMetadataAttribute attribute : getCustomMetadataMap().values()) { attribute.toXContent(builder, params); } builder.endObject(); + diffManifest.toXContent(builder, params); } else if (onOrAfterCodecVersion(CODEC_V1)) { builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); @@ -544,6 +599,8 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws public static class Builder { private String globalMetadataFileName; + private UploadedMetadataAttribute discoveryNodesMetadata; + private UploadedMetadataAttribute clusterBlocksMetadata; private UploadedMetadataAttribute coordinationMetadata; private UploadedMetadataAttribute settingsMetadata; private UploadedMetadataAttribute templatesMetadata; @@ -559,6 +616,7 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; + private ClusterDiffManifest diffManifest; public Builder indices(List indices) { this.indices = indices; @@ -649,6 +707,26 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { return this; } + public Builder diffManifest(ClusterDiffManifest diffManifest) { + this.diffManifest = diffManifest; + return this; + } + + public Builder discoveryNodesMetadata(UploadedMetadataAttribute discoveryNodesMetadata) { + this.discoveryNodesMetadata = discoveryNodesMetadata; + return this; + } + + public Builder clusterBlocksMetadata(UploadedMetadataAttribute clusterBlocksMetadata) { + this.clusterBlocksMetadata = clusterBlocksMetadata; + return this; + } + + public Builder diffManifest(ClusterDiffManifest diffManifest) { + this.diffManifest = diffManifest; + return this; + } + public Builder() { indices = new ArrayList<>(); customMetadataMap = new HashMap<>(); @@ -671,6 +749,7 @@ public Builder(ClusterMetadataManifest manifest) { this.indices = new ArrayList<>(manifest.indices); this.previousClusterUUID = manifest.previousClusterUUID; this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; + this.diffManifest = manifest.diffManifest; } public ClusterMetadataManifest build() { @@ -690,7 +769,10 @@ public ClusterMetadataManifest build() { coordinationMetadata, settingsMetadata, templatesMetadata, - customMetadataMap + customMetadataMap, + discoveryNodesMetadata, + clusterBlocksMetadata, + diffManifest ); } @@ -910,4 +992,94 @@ public String toString() { + '}'; } } + + public static class ClusterDiffManifest implements ToXContentObject { + private final String fromStateUUID; + private final String toStateUUID; + private final boolean coordinationMetadataUpdated; + private final boolean settingsMetadataUpdated; + private final boolean templatesMetadataUpdated; + private final Map customMetadataUpdated; + private final List indicesUpdated; + private final List indicesDeleted; + private final boolean clusterBlocksUpdated; + private final boolean discoveryNodesUpdated; + + ClusterDiffManifest(ClusterState state, ClusterState previousState) { + fromStateUUID = previousState.stateUUID(); + toStateUUID = state.stateUUID(); + coordinationMetadataUpdated = Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); + settingsMetadataUpdated = Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); + templatesMetadataUpdated = Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); + indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); + indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); + clusterBlocksUpdated = state.blocks().equals(previousState.blocks()); + discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); + customMetadataUpdated = new HashMap<>(); + for (String custom : state.metadata().customs().keySet()) { + customMetadataUpdated.put( + custom, + state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) + ); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("from_state_uuid", fromStateUUID); + builder.field("to_state_uuid", toStateUUID); + builder.startObject("metadata_diff"); + { + builder.field("coordination_metadata_diff", coordinationMetadataUpdated); + builder.field("settings_metadata_diff", settingsMetadataUpdated); + builder.field("templates_metadata_diff", templatesMetadataUpdated); + builder.startObject("indices_diff"); + builder.startArray("upserts"); + for (String index : indicesUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray("deletes"); + for (String index : indicesDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + for (Map.Entry entry : customMetadataUpdated.entrySet()) { + if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); + } + } + builder.endObject(); + builder.field("cluster_blocks_diff", clusterBlocksUpdated); + builder.field("discovery_nodes_diff", discoveryNodesUpdated); + } + builder.endObject(); + return builder; + } + + public List findRemovedIndices(Map indices, Map previousIndices) { + List removedIndices = new ArrayList<>(); + for (String index : previousIndices.keySet()) { + // index present in previous state but not in current + if (!indices.containsKey(index)) { + removedIndices.add(index); + } + } + return removedIndices; + } + + public List findUpdatedIndices(Map indices, Map previousIndices) { + List updatedIndices = new ArrayList<>(); + for (String index : indices.keySet()) { + if (!previousIndices.containsKey(index)) { + updatedIndices.add(index); + } else if (previousIndices.get(index).getVersion() != indices.get(index).getVersion()) { + updatedIndices.add(index); + } + } + return updatedIndices; + } + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java new file mode 100644 index 0000000000000..1a1e7786256c0 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; + +public class RemoteClusterStateAttributesManager { + public static final String DISCOVERY_NODES = "nodes"; + public static final String CLUSTER_BLOCKS = "blocks"; + public static final String CUSTOM_PREFIX = "custom"; + public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( + "nodes", + METADATA_NAME_FORMAT, + DiscoveryNodes::fromXContent + ); + public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( + "blocks", + METADATA_NAME_FORMAT, + ClusterBlocks::fromXContent + ); + public static final int CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION = 1; + private final BlobStoreRepository blobStoreRepository; + + RemoteClusterStateAttributesManager(BlobStoreRepository repository) { + blobStoreRepository = repository; + } + + /** + * Allows async upload of Cluster State Attribute components to remote + */ + CheckedRunnable getAsyncMetadataWriteAction( + ClusterState clusterState, + String component, + ChecksumBlobStoreFormat componentMetadataBlobStore, + ToXContent componentData, + LatchedActionListener latchedActionListener + ) { + final BlobContainer remoteStateAttributeContainer = clusterStateAttributeContainer(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()); + final String componentMetadataFilename = clusterStateAttributeFileName(component, clusterState.metadata().version()); + ActionListener completionListener = ActionListener.wrap( + resp -> latchedActionListener.onResponse( + new ClusterMetadataManifest.UploadedMetadataAttribute(component, componentMetadataFilename) + ), + ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) + ); + return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority( + componentData, + remoteStateAttributeContainer, + componentMetadataFilename, + blobStoreRepository.getCompressor(), + completionListener, + FORMAT_PARAMS + ); + } + + private BlobContainer clusterStateAttributeContainer(String clusterName, String clusterUUID) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ + return blobStoreRepository.blobStore() + .blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID)); + } + + private static String clusterStateAttributeFileName(String componentPrefix, Long metadataVersion) { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/______ + return String.join( + DELIMITER, + componentPrefix, + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 54295fe4fb183..eb21359b0512f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -59,6 +59,10 @@ import java.util.stream.Collectors; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; @@ -125,6 +129,7 @@ public class RemoteClusterStateService implements Closeable { private final RemotePersistenceStats remoteStateStats; private RemoteIndexMetadataManager remoteIndexMetadataManager; private RemoteGlobalMetadataManager remoteGlobalMetadataManager; + private RemoteClusterStateAttributesManager remoteClusterStateAttributesManager; private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; private TimeValue staleFileCleanupInterval; @@ -186,6 +191,8 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri clusterState.metadata().customs(), true, true, + true, + true, true ); final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( @@ -196,6 +203,9 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedSettingsMetadata, uploadedMetadataResults.uploadedTemplatesMetadata, uploadedMetadataResults.uploadedCustomMetadataMap, + uploadedMetadataResults.uploadedDiscoveryNodes, + uploadedMetadataResults.uploadedClusterBlocks, + new ClusterMetadataManifest.ClusterDiffManifest(clusterState, ClusterState.EMPTY_STATE), false ); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); @@ -256,6 +266,10 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousClusterState.metadata(), clusterState.metadata() ) == false; + // ToDo: check if these needs to be updated or not + final boolean updateDiscoveryNodes = clusterState.getNodes().delta(previousClusterState.getNodes()).hasChanges(); + final boolean updateClusterBlocks = clusterState.blocks().equals(previousClusterState.blocks()); + final Map previousStateCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); final Map customsToUpload = remoteGlobalMetadataManager.getUpdatedCustoms( clusterState, @@ -302,7 +316,16 @@ public ClusterMetadataManifest writeIncrementalMetadata( // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, // If file is empty and codec is 1 then write global metadata. if (firstUpload) { - uploadedMetadataResults = writeMetadataInParallel(clusterState, toUpload, clusterState.metadata().customs(), true, true, true); + uploadedMetadataResults = writeMetadataInParallel( + clusterState, + toUpload, + clusterState.metadata().customs(), + true, + true, + true, + true, + true + ); } else { uploadedMetadataResults = writeMetadataInParallel( clusterState, @@ -310,7 +333,9 @@ public ClusterMetadataManifest writeIncrementalMetadata( customsToUpload, updateCoordinationMetadata, updateSettingsMetadata, - updateTemplatesMetadata + updateTemplatesMetadata, + updateDiscoveryNodes, + updateClusterBlocks ); } @@ -336,6 +361,9 @@ public ClusterMetadataManifest writeIncrementalMetadata( ? uploadedMetadataResults.uploadedTemplatesMetadata : previousManifest.getTemplatesMetadata(), firstUpload || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), + firstUpload || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), + firstUpload || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), + new ClusterMetadataManifest.ClusterDiffManifest(clusterState, previousClusterState), false ); this.latestClusterName = clusterState.getClusterName().value(); @@ -382,11 +410,12 @@ private UploadedMetadataResults writeMetadataInParallel( Map customToUpload, boolean uploadCoordinationMetadata, boolean uploadSettingsMetadata, - boolean uploadTemplateMetadata + boolean uploadTemplateMetadata, + boolean uploadDiscoveryNodes, + boolean uploadClusterBlock ) throws IOException { int totalUploadTasks = indexToUpload.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata - ? 1 - : 0) + (uploadTemplateMetadata ? 1 : 0); + ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new HashMap<>(totalUploadTasks); Map results = new HashMap<>(totalUploadTasks); @@ -442,6 +471,30 @@ private UploadedMetadataResults writeMetadataInParallel( ) ); } + if (uploadDiscoveryNodes) { + uploadTasks.put( + DISCOVERY_NODES, + remoteClusterStateAttributesManager.getAsyncMetadataWriteAction( + clusterState, + DISCOVERY_NODES, + DISCOVERY_NODES_FORMAT, + clusterState.nodes(), + listener + ) + ); + } + if (uploadClusterBlock) { + uploadTasks.put( + CLUSTER_BLOCKS, + remoteClusterStateAttributesManager.getAsyncMetadataWriteAction( + clusterState, + CLUSTER_BLOCKS, + CLUSTER_BLOCKS_FORMAT, + clusterState.blocks(), + listener + ) + ); + } customToUpload.forEach((key, value) -> { String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key); uploadTasks.put( @@ -521,6 +574,10 @@ private UploadedMetadataResults writeMetadataInParallel( response.uploadedSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (TEMPLATES_METADATA.equals(uploadedMetadata.getComponent())) { response.uploadedTemplatesMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else if (DISCOVERY_NODES.equals(uploadedMetadata.getComponent())) { + response.uploadedDiscoveryNodes = (UploadedMetadataAttribute) uploadedMetadata; + } else if (CLUSTER_BLOCKS.equals(uploadedMetadata.getComponent())) { + response.uploadedClusterBlocks = (UploadedMetadataAttribute) uploadedMetadata; } else { throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); } @@ -548,10 +605,9 @@ private void updateCleanupInterval(TimeValue updatedInterval) { private void cleanUpStaleFiles() { long cleanUpAttemptState = remoteStateStats.getSuccessCount(); - if ( - cleanUpAttemptState - lastCleanupAttemptState > SKIP_CLEANUP_STATE_CHANGES && - this.latestClusterName != null && this.latestClusterUUID != null - ) { + if (cleanUpAttemptState - lastCleanupAttemptState > SKIP_CLEANUP_STATE_CHANGES + && this.latestClusterName != null + && this.latestClusterUUID != null) { logger.info( "Cleaning up stale remote state files for cluster [{}] with uuid [{}]. Last clean was done before {} updates", this.latestClusterName, @@ -588,6 +644,9 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getSettingsMetadata(), previousManifest.getTemplatesMetadata(), previousManifest.getCustomMetadataMap(), + previousManifest.getDiscoveryNodesMetadata(), + previousManifest.getClusterBlocksMetadata(), + previousManifest.getDiffManifest(), true ); deleteStaleClusterUUIDs(clusterState, committedManifest); @@ -626,6 +685,7 @@ public void start() { blobStoreRepository = (BlobStoreRepository) repository; remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings); remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings); + remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(blobStoreRepository); remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); if (isClusterManagerNode) { staleFileDeletionTask = new AsyncStaleFileDeletion(this); @@ -1077,6 +1137,7 @@ public RemotePersistenceStats getStats() { static final class AsyncStaleFileDeletion extends AbstractAsyncTask { private final RemoteClusterStateService remoteClusterStateService; + AsyncStaleFileDeletion(RemoteClusterStateService remoteClusterStateService) { super(logger, remoteClusterStateService.threadpool, remoteClusterStateService.getStaleFileCleanupInterval(), true); this.remoteClusterStateService = remoteClusterStateService; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 4e8fb7cf351ea..3fa9506192188 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -73,19 +73,25 @@ static class UploadedMetadataResults { ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks; public UploadedMetadataResults( List uploadedIndexMetadata, Map uploadedCustomMetadataMap, ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks ) { this.uploadedIndexMetadata = uploadedIndexMetadata; this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; this.uploadedSettingsMetadata = uploadedSettingsMetadata; this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + this.uploadedDiscoveryNodes = uploadedDiscoveryNodes; + this.uploadedClusterBlocks = uploadedClusterBlocks; } public UploadedMetadataResults() { @@ -94,6 +100,8 @@ public UploadedMetadataResults() { this.uploadedCoordinationMetadata = null; this.uploadedSettingsMetadata = null; this.uploadedTemplatesMetadata = null; + this.uploadedDiscoveryNodes = null; + this.uploadedClusterBlocks = null; } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index c741989552462..3523216c8c83f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -129,6 +129,9 @@ ClusterMetadataManifest uploadManifest( ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, Map uploadedCustomMetadataMap, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodesMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocksMetadata, + ClusterMetadataManifest.ClusterDiffManifest clusterDiffManifest, boolean committed ) throws IOException { synchronized (this) { @@ -154,7 +157,10 @@ ClusterMetadataManifest uploadManifest( uploadedCoordinationMetadata, uploadedSettingsMetadata, uploadedTemplatesMetadata, - uploadedCustomMetadataMap + uploadedCustomMetadataMap, + uploadedDiscoveryNodesMetadata, + uploadedClusterBlocksMetadata, + clusterDiffManifest ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 94d71dc9eaa75..ec540779dbe54 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -124,7 +124,10 @@ public void testClusterMetadataManifestXContent() throws IOException { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + null, + null, + null ); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -171,7 +174,10 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), + null, + null, + null ); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 4636d42fb0799..1d48ba4b25af1 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -112,7 +112,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.test.OpenSearchIntegTestCase.client; public class RemoteClusterStateServiceTests extends OpenSearchTestCase { From acdb97e17ad5e4a5dc91d3fedbe274c0c32d2fb4 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Tue, 14 May 2024 23:24:56 +0530 Subject: [PATCH 046/133] Download cluster state from remote --- .../cluster/coordination/Coordinator.java | 7 ++- .../PublicationTransportHandler.java | 42 +++++++++++++- .../coordination/RemotePublishRequest.java | 48 ++++++++++++++++ .../opensearch/discovery/DiscoveryModule.java | 7 ++- .../remote/ClusterMetadataManifest.java | 40 ++++++++++++++ .../remote/RemoteClusterStateService.java | 55 +++++++++++++++++-- .../remote/RemoteGlobalMetadataManager.java | 8 +-- .../gateway/remote/RemoteManifestManager.java | 30 +++++++++- .../main/java/org/opensearch/node/Node.java | 3 +- .../cluster/coordination/NodeJoinTests.java | 3 +- .../PublicationTransportHandlerTests.java | 3 +- .../discovery/DiscoveryModuleTests.java | 3 +- .../snapshots/SnapshotResiliencyTests.java | 3 +- .../AbstractCoordinatorTestCase.java | 3 +- 14 files changed, 233 insertions(+), 22 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 3d74feddfa261..5ca2d93831506 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -84,6 +84,7 @@ import org.opensearch.discovery.PeerFinder; import org.opensearch.discovery.SeedHostsProvider; import org.opensearch.discovery.SeedHostsResolver; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; import org.opensearch.node.remotestore.RemoteStoreNodeService; @@ -207,7 +208,8 @@ public Coordinator( ElectionStrategy electionStrategy, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreNodeService remoteStoreNodeService + RemoteStoreNodeService remoteStoreNodeService, + RemoteClusterStateService remoteClusterStateService ) { this.settings = settings; this.transportService = transportService; @@ -259,7 +261,8 @@ public Coordinator( transportService, namedWriteableRegistry, this::handlePublishRequest, - this::handleApplyCommit + this::handleApplyCommit, + remoteClusterStateService ); this.leaderChecker = new LeaderChecker(settings, clusterSettings, transportService, this::onLeaderFailure, nodeHealthService); this.followersChecker = new FollowersChecker( diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 1fdaeead0d28d..d65d730213ad4 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -31,6 +31,7 @@ package org.opensearch.cluster.coordination; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -47,6 +48,8 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BytesTransportRequest; import org.opensearch.transport.TransportChannel; @@ -97,16 +100,19 @@ public class PublicationTransportHandler { private final TransportRequestOptions stateRequestOptions = TransportRequestOptions.builder() .withType(TransportRequestOptions.Type.STATE) .build(); + private final RemoteClusterStateService remoteClusterStateService; public PublicationTransportHandler( TransportService transportService, NamedWriteableRegistry namedWriteableRegistry, Function handlePublishRequest, - BiConsumer> handleApplyCommit + BiConsumer> handleApplyCommit, + RemoteClusterStateService remoteClusterStateService ) { this.transportService = transportService; this.namedWriteableRegistry = namedWriteableRegistry; this.handlePublishRequest = handlePublishRequest; + this.remoteClusterStateService = remoteClusterStateService; transportService.registerRequestHandler( PUBLISH_STATE_ACTION_NAME, @@ -211,6 +217,40 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque } } + private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest request) throws IOException { + final Optional manifestOptional = remoteClusterStateService.getClusterMetadataManifestByTermVersion(request.getClusterName(), request.getClusterUUID(), request.term, request.version); + if (manifestOptional.isPresent() == false) { + // todo change exception + throw new IncompatibleClusterStateVersionException("No remote state for term version"); + } + ClusterMetadataManifest manifest = manifestOptional.get(); + boolean applyFullState = false; + final ClusterState lastSeen = lastSeenClusterState.get(); + if (lastSeen == null) { + logger.debug("Diff cannot be applied as there is not last cluster state"); + applyFullState = true; + } else if (manifest.getDiffManifest() == null) { + logger.debug("There is no diff in the manifest"); + applyFullState = true; + } else if (manifest.getDiffManifest().getFromStateUUID().equals(lastSeen.stateUUID()) == false) { + logger.debug("Last cluster state not compatible with the diff"); + applyFullState = true; + } else { + ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get()); + final PublishWithJoinResponse response = acceptState(clusterState); + lastSeenClusterState.compareAndSet(lastSeen, clusterState); + return response; + } + + if (applyFullState == true) { + ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest); + logger.debug("Downloaded full cluster state version [{}]", clusterState.version()); + final PublishWithJoinResponse response = acceptState(clusterState); + lastSeenClusterState.set(clusterState); + return response; + } + } + private PublishWithJoinResponse acceptState(ClusterState incomingState) { // if the state is coming from the current node, use original request instead (see currentPublishRequestToSelf for explanation) if (transportService.getLocalNode().equals(incomingState.nodes().getClusterManagerNode())) { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java new file mode 100644 index 0000000000000..b1981c0b4b1a0 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.coordination; + +import java.io.IOException; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +public class RemotePublishRequest extends TermVersionRequest { + + // todo Do we need cluster name and UUID ? + private final String clusterName; + private final String clusterUUID; + + public RemotePublishRequest(DiscoveryNode sourceNode, long term, long version, String clusterName, String clusterUUID) { + super(sourceNode, term, version); + this.clusterName = clusterName; + this.clusterUUID = clusterUUID; + } + + public RemotePublishRequest(StreamInput in) throws IOException { + super(in); + this.clusterName = in.readString(); + this.clusterUUID = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(clusterName); + out.writeString(clusterUUID); + } + + public String getClusterName() { + return clusterName; + } + + public String getClusterUUID() { + return clusterUUID; + } +} diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index 288371aa240a0..2edb9a61a85a8 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -52,6 +52,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.gateway.GatewayMetaState; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.plugins.DiscoveryPlugin; @@ -133,7 +134,8 @@ public DiscoveryModule( RerouteService rerouteService, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreNodeService remoteStoreNodeService + RemoteStoreNodeService remoteStoreNodeService, + RemoteClusterStateService remoteClusterStateService ) { final Collection> joinValidators = new ArrayList<>(); final Map> hostProviders = new HashMap<>(); @@ -211,7 +213,8 @@ public DiscoveryModule( electionStrategy, nodeHealthService, persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + remoteClusterStateService ); } else { throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]"); diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index dccb11f25e5ef..bd189461620c1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -1081,5 +1081,45 @@ public List findUpdatedIndices(Map indices, Map getCustomMetadataUpdated() { + return customMetadataUpdated; + } + + public List getIndicesUpdated() { + return indicesUpdated; + } + + public List getIndicesDeleted() { + return indicesDeleted; + } + + public boolean isClusterBlocksUpdated() { + return clusterBlocksUpdated; + } + + public boolean isDiscoveryNodesUpdated() { + return discoveryNodesUpdated; + } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index eb21359b0512f..a31e336407e8c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -13,8 +13,10 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -29,6 +31,7 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; +import org.opensearch.gateway.remote.ClusterMetadataManifest.ClusterDiffManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -664,6 +667,10 @@ public Optional getLatestClusterMetadataManifest(String return remoteManifestManager.getLatestClusterMetadataManifest(clusterName, clusterUUID); } + public Optional getClusterMetadataManifestByTermVersion(String clusterName, String clusterUUID, long term, long version) { + return remoteManifestManager.getClusterMetadataManifestByTermVersion(clusterName, clusterUUID, term, version); + } + @Override public void close() throws IOException { if (staleFileDeletionTask != null) { @@ -728,25 +735,65 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID ); } + return getClusterStateForManifest(clusterName, clusterMetadataManifest.get()); + } + + public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest) { + // todo make this async // Fetch Global Metadata - Metadata globalMetadata = remoteGlobalMetadataManager.getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get()); + Metadata globalMetadata = remoteGlobalMetadataManager.getGlobalMetadata(clusterName, manifest.getClusterUUID(), manifest); // Fetch Index Metadata Map indices = remoteIndexMetadataManager.getIndexMetadataMap( clusterName, - clusterUUID, - clusterMetadataManifest.get() + manifest.getClusterUUID(), + manifest ); Map indexMetadataMap = new HashMap<>(); indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); }); return ClusterState.builder(ClusterState.EMPTY_STATE) - .version(clusterMetadataManifest.get().getStateVersion()) + .version(manifest.getStateVersion()) .metadata(Metadata.builder(globalMetadata).indices(indexMetadataMap).build()) .build(); } + public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState) { + assert manifest.getDiffManifest() != null; + ClusterDiffManifest diff = manifest.getDiffManifest(); + ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); + Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); + + for (String index:diff.getIndicesUpdated()) { + //todo optimize below iteration + Optional uploadedIndexMetadataOptional = manifest.getIndices().stream().filter(idx -> idx.getIndexName().equals(index)).findFirst(); + assert uploadedIndexMetadataOptional.isPresent() == true; + IndexMetadata indexMetadata = remoteIndexMetadataManager.getIndexMetadata(clusterName, manifest.getClusterUUID(), uploadedIndexMetadataOptional.get()); + metadataBuilder.put(indexMetadata, false); + } + for (String index:diff.getIndicesDeleted()) { + metadataBuilder.remove(index); + } + if (diff.isCoordinationMetadataUpdated()) { + CoordinationMetadata coordinationMetadata = remoteGlobalMetadataManager.getCoordinationMetadata(clusterName, manifest.getClusterUUID(), manifest.getCoordinationMetadata().getUploadedFilename()); + metadataBuilder.coordinationMetadata(coordinationMetadata); + } + if (diff.isSettingsMetadataUpdated()) { + Settings settings = remoteGlobalMetadataManager.getSettingsMetadata(clusterName, manifest.getClusterUUID(), manifest.getSettingsMetadata().getUploadedFilename()); + metadataBuilder.persistentSettings(settings); + } + if (diff.isTemplatesMetadataUpdated()) { + TemplatesMetadata templatesMetadata = remoteGlobalMetadataManager.getTemplatesMetadata(clusterName, manifest.getClusterUUID(), manifest.getTemplatesMetadata().getUploadedFilename()); + metadataBuilder.templates(templatesMetadata); + } + for (String customType : diff.getCustomMetadataUpdated().keySet()) { + Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(clusterName, manifest.getClusterUUID(), manifest.getCustomMetadataMap().get(customType).getUploadedFilename(), customType); + metadataBuilder.putCustom(customType, custom); + } + return clusterStateBuilder.metadata(metadataBuilder).build(); + } + /** * Fetch the previous cluster UUIDs from remote state store and return the most recent valid cluster UUID * diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 9141085b6d5a2..12fb618152639 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -179,7 +179,7 @@ Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetada } } - private CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { + public CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { try { // Fetch Coordination metadata if (coordinationMetadataFileName != null) { @@ -200,7 +200,7 @@ private CoordinationMetadata getCoordinationMetadata(String clusterName, String } } - private Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { + public Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { try { // Fetch Settings metadata if (settingsMetadataFileName != null) { @@ -221,7 +221,7 @@ private Settings getSettingsMetadata(String clusterName, String clusterUUID, Str } } - private TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { + public TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { try { // Fetch Templates metadata if (templatesMetadataFileName != null) { @@ -242,7 +242,7 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste } } - private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { + public Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { requireNonNull(customMetadataFileName); try { // Fetch Custom metadata diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 3523216c8c83f..6f67d6d3285dd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -227,6 +227,11 @@ public Optional getLatestClusterMetadataManifest(String return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); } + public Optional getClusterMetadataManifestByTermVersion(String clusterName, String clusterUUID, long term, long version) { + Optional manifestFileName = getManifestFileNameByTermVersion(clusterName, clusterUUID, term, version); + return manifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); + } + /** * Fetch ClusterMetadataManifest from remote state store * @@ -313,7 +318,7 @@ private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploa * @param limit max no of files to fetch * @return all manifest file names */ - private List getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException { + private List getManifestFileNames(String clusterName, String clusterUUID, String filePrefix, int limit) throws IllegalStateException { try { /* @@ -322,7 +327,7 @@ private List getManifestFileNames(String clusterName, String clust when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top. */ return manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder( - MANIFEST_FILE_PREFIX + DELIMITER, + filePrefix, limit, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC ); @@ -345,6 +350,15 @@ static String getManifestFileName(long term, long version, boolean committed, in ); } + static String getManifestFilePrefixForTermVersion(long term, long version) { + return String.join( + DELIMITER, + MANIFEST_FILE_PREFIX, + RemoteStoreUtils.invertLong(term), + RemoteStoreUtils.invertLong(version) + ) + DELIMITER; + } + /** * Fetch latest ClusterMetadataManifest file from remote state store * @@ -353,11 +367,21 @@ static String getManifestFileName(long term, long version, boolean committed, in * @return latest ClusterMetadataManifest filename */ private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { - List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, 1); + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, MANIFEST_FILE_PREFIX + DELIMITER, 1); if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { return Optional.of(manifestFilesMetadata.get(0).name()); } logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); return Optional.empty(); } + + private Optional getManifestFileNameByTermVersion(String clusterName, String clusterUUID, long term, long version) throws IllegalStateException { + final String filePrefix = getManifestFilePrefixForTermVersion(term, version); + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, filePrefix, 1); + if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { + return Optional.of(manifestFilesMetadata.get(0).name()); + } + logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}, term: {}, version: {}", clusterName, clusterUUID, term, version); + return Optional.empty(); + } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 7fa2b6c8ff497..70204a8e46a28 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1133,7 +1133,8 @@ protected Node( rerouteService, fsHealthService, persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + remoteClusterStateService ); final SearchPipelineService searchPipelineService = new SearchPipelineService( clusterService, diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java index d94f3fb304fe2..e1fffb3a49163 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java @@ -270,7 +270,8 @@ protected void onSendRequest( ElectionStrategy.DEFAULT_INSTANCE, nodeHealthService, persistedStateRegistry, - Mockito.mock(RemoteStoreNodeService.class) + Mockito.mock(RemoteStoreNodeService.class), + null ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 6d94054afdea2..94b9f8b71c362 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -76,7 +76,8 @@ public void testDiffSerializationFailure() { transportService, writableRegistry(), pu -> null, - (pu, l) -> {} + (pu, l) -> {}, + null ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index b33ebf8333b36..0b951c1927c71 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -128,7 +128,8 @@ private DiscoveryModule newModule(Settings settings, List plugi mock(RerouteService.class), null, new PersistedStateRegistry(), - remoteStoreNodeService + remoteStoreNodeService, + null ); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 4326e5fc63961..2eaa19e8cd7d0 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2551,7 +2551,8 @@ public void start(ClusterState initialState) { ElectionStrategy.DEFAULT_INSTANCE, () -> new StatusInfo(HEALTHY, "healthy-info"), persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + null ); clusterManagerService.setClusterStatePublisher(coordinator); coordinator.start(); diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 0754cc1793dc8..0cf9858be7632 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -1180,7 +1180,8 @@ protected Optional getDisruptableMockTransport(Transpo getElectionStrategy(), nodeHealthService, persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + null ); clusterManagerService.setClusterStatePublisher(coordinator); final GatewayService gatewayService = new GatewayService( From 2025f70be843c13f392e334a593bda28dd92bb40 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 15 May 2024 02:30:35 +0530 Subject: [PATCH 047/133] Fix default time interval Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateCleanupManager.java | 2 +- .../gateway/remote/RemoteClusterStateService.java | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 9ac70ae2a2bfe..ef40be2502199 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -50,7 +50,7 @@ public class RemoteClusterStateCleanupManager implements Closeable { public static final int RETAINED_MANIFESTS = 10; public static final int SKIP_CLEANUP_STATE_CHANGES = 10; - public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT = TimeValue.timeValueMillis(60000); + public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT = TimeValue.timeValueMinutes(5); public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM = TimeValue.MINUS_ONE; /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index e2494d836d33a..237f1f7a6abf2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -741,16 +741,6 @@ BlobStore getBlobStore() { return blobStoreRepository.blobStore(); } - private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) { - final Optional latestManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID); - if (!latestManifest.isPresent()) { - final String previousClusterUUID = getLastKnownUUIDFromRemote(clusterName); - assert !clusterUUID.equals(previousClusterUUID) : "Last cluster UUID is same current cluster UUID"; - return previousClusterUUID; - } - return latestManifest.get().getPreviousClusterUUID(); - } - private BlobContainer indexMetadataContainer(String clusterName, String clusterUUID, String indexUUID) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX return blobStoreRepository.blobStore() From b09bce7a71ff4fbe02d53b1b325b6987f2d06415 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 15 May 2024 14:49:58 +0530 Subject: [PATCH 048/133] Merging from remote cluster state --- .../cluster/metadata/IndexMetadata.java | 4 + .../remote/RemoteRoutingTableService.java | 94 +++++++++++-------- .../remote/ClusterMetadataManifest.java | 12 +-- .../remote/RemoteClusterStateService.java | 34 +++++-- .../gateway/remote/RemoteManifestManager.java | 8 +- .../index/remote/RemoteIndexPathUploader.java | 3 - .../main/java/org/opensearch/node/Node.java | 3 +- 7 files changed, 93 insertions(+), 65 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 80b78cfe154f1..eae59f0049a66 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -753,6 +753,10 @@ public String getIndexUUID() { return index.getUUID(); } + public String getIndexName() { + return index.getName(); + } + /** * Test whether the current index UUID is the same as the given one. Returns true if either are _na_ */ diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 43fe2186dea09..230178252b81f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -15,14 +15,19 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.io.stream.BytesStreamOutput; + +import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStream; +import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -30,6 +35,10 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.io.*; +import java.io.Closeable; +import java.io.IOException; + +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -37,6 +46,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; /** @@ -55,6 +65,9 @@ public class RemoteRoutingTableService implements Closeable { Setting.Property.NodeScope, Setting.Property.Final ); + public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing"; + public static final String DELIMITER = "__"; + private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); private final Settings settings; private final Supplier repositoriesService; @@ -71,19 +84,15 @@ public RemoteRoutingTableService(Supplier repositoriesServi } public List writeFullRoutingTable(ClusterState clusterState, String previousClusterUUID) { + + //batch index count and parallelize RoutingTable currentRoutingTable = clusterState.getRoutingTable(); List uploadedIndices = new ArrayList<>(); - for(IndexRoutingTable indexRouting: currentRoutingTable.getIndicesRouting().values()){ - try { - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting, currentRoutingTable.version(), Version.CURRENT); - BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add("routing-table")); - container.writeBlobWithMetadata(indexRouting.getIndex().getName(), indexRoutingStream, indexRoutingStream.read(), true, null); - logger.info("SUccessful write"); - uploadedIndices.add(new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), "dummyfilename")); - }catch (IOException e) { - logger.error("Failed to write {}", e); - } + BlobPath custerMetadataBasePath = getCusterMetadataBasePath(blobStoreRepository, clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID()); + for (IndexRoutingTable indexRouting : currentRoutingTable.getIndicesRouting().values()) { + uploadedIndices.add(uploadIndex(indexRouting, clusterState.getRoutingTable().version(), custerMetadataBasePath)); } logger.info("uploadedIndices {}", uploadedIndices); @@ -94,49 +103,54 @@ public List writeIncrementalRouti ClusterState previousClusterState, ClusterState clusterState, ClusterMetadataManifest previousManifest) { + final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() .stream() .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); logger.info("allUploadedIndicesRouting ROUTING {}", allUploadedIndicesRouting); + Map previousIndexRoutingTable = previousClusterState.getRoutingTable().getIndicesRouting(); List uploadedIndices = new ArrayList<>(); - for(IndexRoutingTable indexRouting: clusterState.getRoutingTable().getIndicesRouting().values()){ - if(previousClusterState.getRoutingTable().getIndicesRouting().containsKey(indexRouting.getIndex().getName())) { + BlobPath custerMetadataBasePath = getCusterMetadataBasePath(blobStoreRepository, clusterState.getClusterName().value(), + clusterState.metadata().clusterUUID()); + for (IndexRoutingTable indexRouting : clusterState.getRoutingTable().getIndicesRouting().values()) { + if (previousIndexRoutingTable.containsKey(indexRouting.getIndex().getName()) && indexRouting.equals(previousIndexRoutingTable.get(indexRouting.getIndex().getName()))) { logger.info("index exists {}", indexRouting.getIndex().getName()); - //existing index, check if shards are changed - - IndexRoutingTable previousIndexRouting = previousClusterState.getRoutingTable().getIndicesRouting().get(indexRouting.getIndex().getName()); - if (indexRouting.equals(previousIndexRouting)){ - uploadedIndices.add(allUploadedIndicesRouting.get(indexRouting.getIndex().getName())); - continue; - } - try { - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting, clusterState.getRoutingTable().version(), Version.CURRENT); - BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add("routing-table").add(indexRouting.getIndex().getName()).add(String.valueOf(clusterState.getRoutingTable().version()))); - container.writeBlob(indexRouting.getIndex().getName(), indexRoutingStream, 4096,true); - logger.info("SUccessful write"); - uploadedIndices.add(new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString())); - } catch (IOException e) { - logger.error("Failed to write {}", e); - } + //existing index with no shard change. + uploadedIndices.add(allUploadedIndicesRouting.get(indexRouting.getIndex().getName())); } else { - // new index upload - logger.info("cuurent version {}, previous sversiion {}", clusterState.getRoutingTable().version(), previousClusterState.getRoutingTable().version()); - try { - InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting, clusterState.getRoutingTable().version(), Version.CURRENT); - BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add("routing-table").add(indexRouting.getIndex().getName()).add(String.valueOf(clusterState.getRoutingTable().version()))); - container.writeBlob(indexRouting.getIndex().getName(), indexRoutingStream, 4096,true); - //container.writeBlobWithMetadata(indexRouting.getIndex().getName(), indexRoutingStream, indexRoutingStream.read(), true, null); - logger.info("SUccessful write"); - uploadedIndices.add(new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString())); - } catch (IOException e) { - logger.error("Failed to write {}", e); - } + // new index or shards changed, in both cases we upload new index file. + uploadedIndices.add(uploadIndex(indexRouting, clusterState.getRoutingTable().version(), custerMetadataBasePath)); } } return uploadedIndices; } + private ClusterMetadataManifest.UploadedIndexMetadata uploadIndex(IndexRoutingTable indexRouting, long routingTableVersion, BlobPath custerMetadataBasePath) { + try { + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting); + BlobContainer container = blobStoreRepository.blobStore().blobContainer(custerMetadataBasePath.add(INDEX_ROUTING_PATH_TOKEN).add(indexRouting.getIndex().getUUID())); + + container.writeBlob(getIndexRoutingFileName(), indexRoutingStream, 4096, true); + return new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString() + getIndexRoutingFileName()); + + } catch (IOException e) { + logger.error("Failed to write {}", e); + } + logger.info("SUccessful write"); + return null; + } + + private String getIndexRoutingFileName() { + return String.join( + DELIMITER, + //RemoteStoreUtils.invertLong(indexMetadata.getVersion()), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf("CODEC1") // Keep the codec version at last place only, during read we reads last + // place to determine codec version. + ); + + } public RoutingTable getLatestRoutingTable(String clusterName, String clusterUUID) { return null; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 8db84f7cd5b45..a2082b7a5af67 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -203,10 +203,10 @@ private static Map customMetadata(Object[] fi fields -> manifestV2Builder(fields).build() ); - private static final ConstructingObjectParser PARSER_V3 = new ConstructingObjectParser<>( - "cluster_metadata_manifest", - fields -> manifestV3Builder(fields).build() - ); + private static final ConstructingObjectParser PARSER_V3 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> manifestV3Builder(fields).build() + ); private static final ConstructingObjectParser PARSER_V4 = new ConstructingObjectParser<>( "cluster_metadata_manifest", @@ -587,7 +587,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } - if (onOrAfterCodecVersion(CODEC_V3)) { + if (onOrAfterCodecVersion(CODEC_V4)) { builder.field(ROUTING_TABLE_VERSION_FIELD.getPreferredName(), getRoutingTableVersion()); builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { @@ -1144,7 +1144,6 @@ public static class ClusterDiffManifest implements ToXContentObject { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); { builder.field("from_state_uuid", fromStateUUID); builder.field("to_state_uuid", toStateUUID); @@ -1173,7 +1172,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("cluster_blocks_diff", clusterBlocksUpdated); builder.field("discovery_nodes_diff", discoveryNodesUpdated); } - builder.endObject(); return builder; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 45418dd9128a4..48efcfd800e7c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -16,7 +16,6 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; -import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -39,7 +38,6 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; @@ -228,8 +226,11 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedDiscoveryNodes, uploadedMetadataResults.uploadedClusterBlocks, new ClusterMetadataManifest.ClusterDiffManifest(clusterState, ClusterState.EMPTY_STATE), + routingIndexMetadata, false ); + + logger.info("MANIFEST IN FULL STATE {}", manifest); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); @@ -366,6 +367,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( updateClusterBlocks ); } + List routingIndexMetadata = new ArrayList<>(); + if(remoteRoutingTableService!=null) { + routingIndexMetadata = remoteRoutingTableService.writeIncrementalRoutingTable(previousClusterState, clusterState, previousManifest); + logger.info("routingIndexMetadata incremental {}", routingIndexMetadata); + } // update the map if the metadata was uploaded uploadedMetadataResults.uploadedIndexMetadata.forEach( @@ -376,11 +382,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousStateCustomMap.keySet().forEach(allUploadedCustomMap::remove); previousStateIndexMetadataVersionByName.keySet().forEach(allUploadedIndexMetadata::remove); - List routingIndexMetadata = new ArrayList<>(); - if(remoteRoutingTableService!=null) { - routingIndexMetadata = remoteRoutingTableService.writeIncrementalRoutingTable(previousClusterState, clusterState, previousManifest); - logger.info("routingIndexMetadata incremental {}", routingIndexMetadata); - } + final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), @@ -398,8 +400,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUpload || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), firstUpload || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), new ClusterMetadataManifest.ClusterDiffManifest(clusterState, previousClusterState), - false + routingIndexMetadata, false ); + + logger.info("MANIFEST IN INC STATE {}", manifest); + this.latestClusterName = clusterState.getClusterName().value(); this.latestClusterUUID = clusterState.metadata().clusterUUID(); @@ -593,7 +598,9 @@ private UploadedMetadataResults writeMetadataInParallel( } UploadedMetadataResults response = new UploadedMetadataResults(); results.forEach((name, uploadedMetadata) -> { - if (name.contains(CUSTOM_METADATA)) { + if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class)) { + response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); + } else if (name.contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; response.uploadedCustomMetadataMap.put( @@ -681,7 +688,7 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getDiscoveryNodesMetadata(), previousManifest.getClusterBlocksMetadata(), previousManifest.getDiffManifest(), - true + previousManifest.getIndicesRouting(), true ); deleteStaleClusterUUIDs(clusterState, committedManifest); return committedManifest; @@ -723,6 +730,13 @@ public void start() { if(this.remoteRoutingTableService != null) { this.remoteRoutingTableService.start(); } + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings); + remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(blobStoreRepository); + remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); + if (isClusterManagerNode) { + staleFileDeletionTask = new AsyncStaleFileDeletion(this); + } } private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 3523216c8c83f..c5ff35fdff699 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -44,7 +44,7 @@ public class RemoteManifestManager { public static final String MANIFEST_PATH_TOKEN = "manifest"; public static final String MANIFEST_FILE_PREFIX = "manifest"; public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V4; public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); @@ -132,7 +132,7 @@ ClusterMetadataManifest uploadManifest( ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodesMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocksMetadata, ClusterMetadataManifest.ClusterDiffManifest clusterDiffManifest, - boolean committed + List routingIndexMetadata, boolean committed ) throws IOException { synchronized (this) { final String manifestFileName = getManifestFileName( @@ -160,7 +160,9 @@ ClusterMetadataManifest uploadManifest( uploadedCustomMetadataMap, uploadedDiscoveryNodesMetadata, uploadedClusterBlocksMetadata, - clusterDiffManifest + clusterDiffManifest, + clusterState.getRoutingTable().version(), + routingIndexMetadata ); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index d736a82d57a7c..ff2114de54398 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -45,7 +45,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING; import static org.opensearch.index.remote.RemoteIndexPath.COMBINED_PATH; import static org.opensearch.index.remote.RemoteIndexPath.SEGMENT_PATH; import static org.opensearch.index.remote.RemoteIndexPath.TRANSLOG_PATH; @@ -98,8 +97,6 @@ public RemoteIndexPathUploader( // If the remote data attributes are not present, then there is no effect of translog and segment being same or different or null. isTranslogSegmentRepoSame = isTranslogSegmentRepoSame(); Objects.requireNonNull(clusterSettings); - indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); - clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); } @Override diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 614f39166ea66..34445a557a558 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -742,8 +742,7 @@ protected Node( settings, clusterService.getClusterSettings(), threadPool::preciseRelativeTimeInNanos, - threadPool, - List.of(remoteIndexPathUploader) + threadPool ); } else { remoteClusterStateService = null; From df8ddad3f791d9cbe02a479c5d8f5b3eada72cea Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Tue, 14 May 2024 23:24:56 +0530 Subject: [PATCH 049/133] Download cluster state from remote --- .../cluster/coordination/Coordinator.java | 7 ++- .../PublicationTransportHandler.java | 42 +++++++++++++- .../coordination/RemotePublishRequest.java | 48 ++++++++++++++++ .../opensearch/discovery/DiscoveryModule.java | 7 ++- .../remote/ClusterMetadataManifest.java | 40 ++++++++++++++ .../remote/RemoteClusterStateService.java | 55 +++++++++++++++++-- .../remote/RemoteGlobalMetadataManager.java | 8 +-- .../gateway/remote/RemoteManifestManager.java | 30 +++++++++- .../main/java/org/opensearch/node/Node.java | 3 +- .../cluster/coordination/NodeJoinTests.java | 3 +- .../PublicationTransportHandlerTests.java | 3 +- .../discovery/DiscoveryModuleTests.java | 3 +- .../snapshots/SnapshotResiliencyTests.java | 3 +- .../AbstractCoordinatorTestCase.java | 3 +- 14 files changed, 233 insertions(+), 22 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 3d74feddfa261..5ca2d93831506 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -84,6 +84,7 @@ import org.opensearch.discovery.PeerFinder; import org.opensearch.discovery.SeedHostsProvider; import org.opensearch.discovery.SeedHostsResolver; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; import org.opensearch.node.remotestore.RemoteStoreNodeService; @@ -207,7 +208,8 @@ public Coordinator( ElectionStrategy electionStrategy, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreNodeService remoteStoreNodeService + RemoteStoreNodeService remoteStoreNodeService, + RemoteClusterStateService remoteClusterStateService ) { this.settings = settings; this.transportService = transportService; @@ -259,7 +261,8 @@ public Coordinator( transportService, namedWriteableRegistry, this::handlePublishRequest, - this::handleApplyCommit + this::handleApplyCommit, + remoteClusterStateService ); this.leaderChecker = new LeaderChecker(settings, clusterSettings, transportService, this::onLeaderFailure, nodeHealthService); this.followersChecker = new FollowersChecker( diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 1fdaeead0d28d..d65d730213ad4 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -31,6 +31,7 @@ package org.opensearch.cluster.coordination; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -47,6 +48,8 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BytesTransportRequest; import org.opensearch.transport.TransportChannel; @@ -97,16 +100,19 @@ public class PublicationTransportHandler { private final TransportRequestOptions stateRequestOptions = TransportRequestOptions.builder() .withType(TransportRequestOptions.Type.STATE) .build(); + private final RemoteClusterStateService remoteClusterStateService; public PublicationTransportHandler( TransportService transportService, NamedWriteableRegistry namedWriteableRegistry, Function handlePublishRequest, - BiConsumer> handleApplyCommit + BiConsumer> handleApplyCommit, + RemoteClusterStateService remoteClusterStateService ) { this.transportService = transportService; this.namedWriteableRegistry = namedWriteableRegistry; this.handlePublishRequest = handlePublishRequest; + this.remoteClusterStateService = remoteClusterStateService; transportService.registerRequestHandler( PUBLISH_STATE_ACTION_NAME, @@ -211,6 +217,40 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque } } + private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest request) throws IOException { + final Optional manifestOptional = remoteClusterStateService.getClusterMetadataManifestByTermVersion(request.getClusterName(), request.getClusterUUID(), request.term, request.version); + if (manifestOptional.isPresent() == false) { + // todo change exception + throw new IncompatibleClusterStateVersionException("No remote state for term version"); + } + ClusterMetadataManifest manifest = manifestOptional.get(); + boolean applyFullState = false; + final ClusterState lastSeen = lastSeenClusterState.get(); + if (lastSeen == null) { + logger.debug("Diff cannot be applied as there is not last cluster state"); + applyFullState = true; + } else if (manifest.getDiffManifest() == null) { + logger.debug("There is no diff in the manifest"); + applyFullState = true; + } else if (manifest.getDiffManifest().getFromStateUUID().equals(lastSeen.stateUUID()) == false) { + logger.debug("Last cluster state not compatible with the diff"); + applyFullState = true; + } else { + ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get()); + final PublishWithJoinResponse response = acceptState(clusterState); + lastSeenClusterState.compareAndSet(lastSeen, clusterState); + return response; + } + + if (applyFullState == true) { + ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest); + logger.debug("Downloaded full cluster state version [{}]", clusterState.version()); + final PublishWithJoinResponse response = acceptState(clusterState); + lastSeenClusterState.set(clusterState); + return response; + } + } + private PublishWithJoinResponse acceptState(ClusterState incomingState) { // if the state is coming from the current node, use original request instead (see currentPublishRequestToSelf for explanation) if (transportService.getLocalNode().equals(incomingState.nodes().getClusterManagerNode())) { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java new file mode 100644 index 0000000000000..b1981c0b4b1a0 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.coordination; + +import java.io.IOException; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +public class RemotePublishRequest extends TermVersionRequest { + + // todo Do we need cluster name and UUID ? + private final String clusterName; + private final String clusterUUID; + + public RemotePublishRequest(DiscoveryNode sourceNode, long term, long version, String clusterName, String clusterUUID) { + super(sourceNode, term, version); + this.clusterName = clusterName; + this.clusterUUID = clusterUUID; + } + + public RemotePublishRequest(StreamInput in) throws IOException { + super(in); + this.clusterName = in.readString(); + this.clusterUUID = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(clusterName); + out.writeString(clusterUUID); + } + + public String getClusterName() { + return clusterName; + } + + public String getClusterUUID() { + return clusterUUID; + } +} diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index 288371aa240a0..2edb9a61a85a8 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -52,6 +52,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.gateway.GatewayMetaState; +import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.plugins.DiscoveryPlugin; @@ -133,7 +134,8 @@ public DiscoveryModule( RerouteService rerouteService, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreNodeService remoteStoreNodeService + RemoteStoreNodeService remoteStoreNodeService, + RemoteClusterStateService remoteClusterStateService ) { final Collection> joinValidators = new ArrayList<>(); final Map> hostProviders = new HashMap<>(); @@ -211,7 +213,8 @@ public DiscoveryModule( electionStrategy, nodeHealthService, persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + remoteClusterStateService ); } else { throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]"); diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index a2082b7a5af67..2a262cad20c4d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -1197,5 +1197,45 @@ public List findUpdatedIndices(Map indices, Map getCustomMetadataUpdated() { + return customMetadataUpdated; + } + + public List getIndicesUpdated() { + return indicesUpdated; + } + + public List getIndicesDeleted() { + return indicesDeleted; + } + + public boolean isClusterBlocksUpdated() { + return clusterBlocksUpdated; + } + + public boolean isDiscoveryNodesUpdated() { + return discoveryNodesUpdated; + } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 48efcfd800e7c..548beb7eaac83 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -13,9 +13,11 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -30,6 +32,7 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; +import org.opensearch.gateway.remote.ClusterMetadataManifest.ClusterDiffManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -705,6 +708,10 @@ public Optional getLatestClusterMetadataManifest(String return remoteManifestManager.getLatestClusterMetadataManifest(clusterName, clusterUUID); } + public Optional getClusterMetadataManifestByTermVersion(String clusterName, String clusterUUID, long term, long version) { + return remoteManifestManager.getClusterMetadataManifestByTermVersion(clusterName, clusterUUID, term, version); + } + @Override public void close() throws IOException { if (staleFileDeletionTask != null) { @@ -780,25 +787,65 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID ); } + return getClusterStateForManifest(clusterName, clusterMetadataManifest.get()); + } + + public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest) { + // todo make this async // Fetch Global Metadata - Metadata globalMetadata = remoteGlobalMetadataManager.getGlobalMetadata(clusterName, clusterUUID, clusterMetadataManifest.get()); + Metadata globalMetadata = remoteGlobalMetadataManager.getGlobalMetadata(clusterName, manifest.getClusterUUID(), manifest); // Fetch Index Metadata Map indices = remoteIndexMetadataManager.getIndexMetadataMap( clusterName, - clusterUUID, - clusterMetadataManifest.get() + manifest.getClusterUUID(), + manifest ); Map indexMetadataMap = new HashMap<>(); indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); }); return ClusterState.builder(ClusterState.EMPTY_STATE) - .version(clusterMetadataManifest.get().getStateVersion()) + .version(manifest.getStateVersion()) .metadata(Metadata.builder(globalMetadata).indices(indexMetadataMap).build()) .build(); } + public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState) { + assert manifest.getDiffManifest() != null; + ClusterDiffManifest diff = manifest.getDiffManifest(); + ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); + Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); + + for (String index:diff.getIndicesUpdated()) { + //todo optimize below iteration + Optional uploadedIndexMetadataOptional = manifest.getIndices().stream().filter(idx -> idx.getIndexName().equals(index)).findFirst(); + assert uploadedIndexMetadataOptional.isPresent() == true; + IndexMetadata indexMetadata = remoteIndexMetadataManager.getIndexMetadata(clusterName, manifest.getClusterUUID(), uploadedIndexMetadataOptional.get()); + metadataBuilder.put(indexMetadata, false); + } + for (String index:diff.getIndicesDeleted()) { + metadataBuilder.remove(index); + } + if (diff.isCoordinationMetadataUpdated()) { + CoordinationMetadata coordinationMetadata = remoteGlobalMetadataManager.getCoordinationMetadata(clusterName, manifest.getClusterUUID(), manifest.getCoordinationMetadata().getUploadedFilename()); + metadataBuilder.coordinationMetadata(coordinationMetadata); + } + if (diff.isSettingsMetadataUpdated()) { + Settings settings = remoteGlobalMetadataManager.getSettingsMetadata(clusterName, manifest.getClusterUUID(), manifest.getSettingsMetadata().getUploadedFilename()); + metadataBuilder.persistentSettings(settings); + } + if (diff.isTemplatesMetadataUpdated()) { + TemplatesMetadata templatesMetadata = remoteGlobalMetadataManager.getTemplatesMetadata(clusterName, manifest.getClusterUUID(), manifest.getTemplatesMetadata().getUploadedFilename()); + metadataBuilder.templates(templatesMetadata); + } + for (String customType : diff.getCustomMetadataUpdated().keySet()) { + Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(clusterName, manifest.getClusterUUID(), manifest.getCustomMetadataMap().get(customType).getUploadedFilename(), customType); + metadataBuilder.putCustom(customType, custom); + } + return clusterStateBuilder.metadata(metadataBuilder).build(); + } + /** * Fetch the previous cluster UUIDs from remote state store and return the most recent valid cluster UUID * diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 9141085b6d5a2..12fb618152639 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -179,7 +179,7 @@ Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetada } } - private CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { + public CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { try { // Fetch Coordination metadata if (coordinationMetadataFileName != null) { @@ -200,7 +200,7 @@ private CoordinationMetadata getCoordinationMetadata(String clusterName, String } } - private Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { + public Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { try { // Fetch Settings metadata if (settingsMetadataFileName != null) { @@ -221,7 +221,7 @@ private Settings getSettingsMetadata(String clusterName, String clusterUUID, Str } } - private TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { + public TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { try { // Fetch Templates metadata if (templatesMetadataFileName != null) { @@ -242,7 +242,7 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste } } - private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { + public Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { requireNonNull(customMetadataFileName); try { // Fetch Custom metadata diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index c5ff35fdff699..83a4d7f98b284 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -229,6 +229,11 @@ public Optional getLatestClusterMetadataManifest(String return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); } + public Optional getClusterMetadataManifestByTermVersion(String clusterName, String clusterUUID, long term, long version) { + Optional manifestFileName = getManifestFileNameByTermVersion(clusterName, clusterUUID, term, version); + return manifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); + } + /** * Fetch ClusterMetadataManifest from remote state store * @@ -315,7 +320,7 @@ private void setMetadataManifestUploadTimeout(TimeValue newMetadataManifestUploa * @param limit max no of files to fetch * @return all manifest file names */ - private List getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException { + private List getManifestFileNames(String clusterName, String clusterUUID, String filePrefix, int limit) throws IllegalStateException { try { /* @@ -324,7 +329,7 @@ private List getManifestFileNames(String clusterName, String clust when sorted in LEXICOGRAPHIC order the latest uploaded manifest file comes on top. */ return manifestContainer(clusterName, clusterUUID).listBlobsByPrefixInSortedOrder( - MANIFEST_FILE_PREFIX + DELIMITER, + filePrefix, limit, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC ); @@ -347,6 +352,15 @@ static String getManifestFileName(long term, long version, boolean committed, in ); } + static String getManifestFilePrefixForTermVersion(long term, long version) { + return String.join( + DELIMITER, + MANIFEST_FILE_PREFIX, + RemoteStoreUtils.invertLong(term), + RemoteStoreUtils.invertLong(version) + ) + DELIMITER; + } + /** * Fetch latest ClusterMetadataManifest file from remote state store * @@ -355,11 +369,21 @@ static String getManifestFileName(long term, long version, boolean committed, in * @return latest ClusterMetadataManifest filename */ private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { - List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, 1); + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, MANIFEST_FILE_PREFIX + DELIMITER, 1); if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { return Optional.of(manifestFilesMetadata.get(0).name()); } logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); return Optional.empty(); } + + private Optional getManifestFileNameByTermVersion(String clusterName, String clusterUUID, long term, long version) throws IllegalStateException { + final String filePrefix = getManifestFilePrefixForTermVersion(term, version); + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, filePrefix, 1); + if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { + return Optional.of(manifestFilesMetadata.get(0).name()); + } + logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}, term: {}, version: {}", clusterName, clusterUUID, term, version); + return Optional.empty(); + } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 34445a557a558..884dfb38ccc08 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -1144,7 +1144,8 @@ protected Node( rerouteService, fsHealthService, persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + remoteClusterStateService ); final SearchPipelineService searchPipelineService = new SearchPipelineService( clusterService, diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java index d94f3fb304fe2..e1fffb3a49163 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java @@ -270,7 +270,8 @@ protected void onSendRequest( ElectionStrategy.DEFAULT_INSTANCE, nodeHealthService, persistedStateRegistry, - Mockito.mock(RemoteStoreNodeService.class) + Mockito.mock(RemoteStoreNodeService.class), + null ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 6d94054afdea2..94b9f8b71c362 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -76,7 +76,8 @@ public void testDiffSerializationFailure() { transportService, writableRegistry(), pu -> null, - (pu, l) -> {} + (pu, l) -> {}, + null ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index b33ebf8333b36..0b951c1927c71 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -128,7 +128,8 @@ private DiscoveryModule newModule(Settings settings, List plugi mock(RerouteService.class), null, new PersistedStateRegistry(), - remoteStoreNodeService + remoteStoreNodeService, + null ); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 95a343f3b4025..4b1edd6efc1b9 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2552,7 +2552,8 @@ public void start(ClusterState initialState) { ElectionStrategy.DEFAULT_INSTANCE, () -> new StatusInfo(HEALTHY, "healthy-info"), persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + null ); clusterManagerService.setClusterStatePublisher(coordinator); coordinator.start(); diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 0754cc1793dc8..0cf9858be7632 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -1180,7 +1180,8 @@ protected Optional getDisruptableMockTransport(Transpo getElectionStrategy(), nodeHealthService, persistedStateRegistry, - remoteStoreNodeService + remoteStoreNodeService, + null ); clusterManagerService.setClusterStatePublisher(coordinator); final GatewayService gatewayService = new GatewayService( From df3c08832b224cc89deac70b9123948e645cd327 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 15 May 2024 14:51:39 +0530 Subject: [PATCH 050/133] Added parser for ClusterDiffManifest Signed-off-by: Shivansh Arora --- .../opensearch/cluster/RestoreInProgress.java | 1 - .../remote/ClusterMetadataManifest.java | 216 ++++++++++++++++-- 2 files changed, 196 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java index 627a04bd8277b..a1fcbab3cbb21 100644 --- a/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/RestoreInProgress.java @@ -32,7 +32,6 @@ package org.opensearch.cluster; -import org.elasticsearch.snapshots.SnapshotId; import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.cluster.metadata.Metadata; diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index bd189461620c1..0cc8ef63fcb18 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -132,6 +132,18 @@ private static Map customMetadata(Object[] fi return customs.stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())); } + private static UploadedMetadataAttribute discoveryNodesMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[15]; + } + + private static UploadedMetadataAttribute clusterBlocksMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[16]; + } + + private static ClusterDiffManifest diffManifest(Object[] fields) { + return (ClusterDiffManifest) fields[17]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> ClusterMetadataManifest.builder() @@ -188,12 +200,37 @@ private static Map customMetadata(Object[] fi .build() ); - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; + private static final ConstructingObjectParser PARSER_V3 = new ConstructingObjectParser<>( + "cluster_metadata_manifest", + fields -> ClusterMetadataManifest.builder() + .clusterTerm(term(fields)) + .stateVersion(version(fields)) + .clusterUUID(clusterUUID(fields)) + .stateUUID(stateUUID(fields)) + .opensearchVersion(opensearchVersion(fields)) + .nodeId(nodeId(fields)) + .committed(committed(fields)) + .codecVersion(codecVersion(fields)) + .indices(indices(fields)) + .previousClusterUUID(previousClusterUUID(fields)) + .clusterUUIDCommitted(clusterUUIDCommitted(fields)) + .coordinationMetadata(coordinationMetadata(fields)) + .settingMetadata(settingsMetadata(fields)) + .templatesMetadata(templatesMetadata(fields)) + .customMetadataMap(customMetadata(fields)) + .discoveryNodesMetadata(discoveryNodesMetadata(fields)) + .clusterBlocksMetadata(clusterBlocksMetadata(fields)) + .diffManifest(diffManifest(fields)) + .build() + ); + + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V3; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); declareParser(PARSER_V2, CODEC_V2); + declareParser(PARSER_V3, CODEC_V3); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -238,6 +275,9 @@ private static void declareParser(ConstructingObjectParser= CODEC_V3) { + parser.declareNamedObject(ConstructingObjectParser.constructorArg(), ClusterDiffManifest.PARSER, DIFF_MANIFEST); + } } private final int codecVersion; @@ -707,11 +747,6 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { return this; } - public Builder diffManifest(ClusterDiffManifest diffManifest) { - this.diffManifest = diffManifest; - return this; - } - public Builder discoveryNodesMetadata(UploadedMetadataAttribute discoveryNodesMetadata) { this.discoveryNodesMetadata = discoveryNodesMetadata; return this; @@ -994,6 +1029,46 @@ public String toString() { } public static class ClusterDiffManifest implements ToXContentObject { + private static final ParseField FROM_STATE_UUID_FIELD = new ParseField("from_state_uuid"); + private static final ParseField TO_STATE_UUID_FIELD = new ParseField("to_state_uuid"); + private static final ParseField METADATA_DIFF_FIELD = new ParseField("metadata_diff"); + private static final ParseField COORDINATION_METADATA_UPDATED_FIELD = new ParseField( + "coordination_metadata_diff" + ); + private static final ParseField SETTINGS_METADATA_UPDATED_FIELD = new ParseField("settings_metadata_diff"); + private static final ParseField TEMPLATES_METADATA_UPDATED_FIELD = new ParseField("templates_metadata_diff"); + private static final ParseField INDICES_DIFF_FIELD = new ParseField("indices_diff"); + private static final ParseField UPSERTS_FIELD = new ParseField("upserts"); + private static final ParseField DELETES_FIELD = new ParseField("deletes"); + private static final ParseField CLUSTER_BLOCKS_UPDATED_FIELD = new ParseField("cluster_blocks_diff"); + private static final ParseField DISCOVERY_NODES_UPDATED_FIELD = new ParseField("discovery_nodes_diff"); + private static final ObjectParser.NamedObjectParser PARSER; + static { + ConstructingObjectParser innerParser = new ConstructingObjectParser( + "cluster_diff_manifest", + fields -> ClusterDiffManifest.builder() + .fromStateUUID((String) fields[0]) + .toStateUUID((String) fields[1]) + .coordinationMetadataUpdated((Boolean) fields[2]) + .settingsMetadataUpdated((Boolean) fields[3]) + .templatesMetadataUpdated((Boolean) fields[4]) + .indicesUpdated((List) fields[5]) + .indicesDeleted((List) fields[6]) + .clusterBlocksUpdated((Boolean) fields[7]) + .discoveryNodesUpdated((Boolean) fields[8]) + .build() + ); + innerParser.declareString(ConstructingObjectParser.constructorArg(), FROM_STATE_UUID_FIELD); + innerParser.declareString(ConstructingObjectParser.constructorArg(), TO_STATE_UUID_FIELD); + innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), COORDINATION_METADATA_UPDATED_FIELD); + innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), SETTINGS_METADATA_UPDATED_FIELD); + innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), TEMPLATES_METADATA_UPDATED_FIELD); + innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), UPSERTS_FIELD); + innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), DELETES_FIELD); + innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), DISCOVERY_NODES_UPDATED_FIELD); + innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_BLOCKS_UPDATED_FIELD); + PARSER = ((p, c, name) -> innerParser.parse(p, null)); + } private final String fromStateUUID; private final String toStateUUID; private final boolean coordinationMetadataUpdated; @@ -1024,41 +1099,59 @@ public static class ClusterDiffManifest implements ToXContentObject { } } + public ClusterDiffManifest(String fromStateUUID, String toStateUUID, boolean coordinationMetadataUpdated, boolean settingsMetadataUpdated, boolean templatesMetadataUpdated, Map customMetadataUpdated, List indicesUpdated, List indicesDeleted, boolean clusterBlocksUpdated, boolean discoveryNodesUpdated) { + this.fromStateUUID = fromStateUUID; + this.toStateUUID = toStateUUID; + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + this.settingsMetadataUpdated = settingsMetadataUpdated; + this.templatesMetadataUpdated = templatesMetadataUpdated; + this.customMetadataUpdated = customMetadataUpdated; + this.indicesUpdated = indicesUpdated; + this.indicesDeleted = indicesDeleted; + this.clusterBlocksUpdated = clusterBlocksUpdated; + this.discoveryNodesUpdated = discoveryNodesUpdated; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - builder.field("from_state_uuid", fromStateUUID); - builder.field("to_state_uuid", toStateUUID); - builder.startObject("metadata_diff"); + builder.field(FROM_STATE_UUID_FIELD.getPreferredName(), fromStateUUID); + builder.field(TO_STATE_UUID_FIELD.getPreferredName(), toStateUUID); + builder.startObject(METADATA_DIFF_FIELD.getPreferredName()); { - builder.field("coordination_metadata_diff", coordinationMetadataUpdated); - builder.field("settings_metadata_diff", settingsMetadataUpdated); - builder.field("templates_metadata_diff", templatesMetadataUpdated); - builder.startObject("indices_diff"); - builder.startArray("upserts"); + builder.field(COORDINATION_METADATA_UPDATED_FIELD.getPreferredName(), coordinationMetadataUpdated); + builder.field(SETTINGS_METADATA_UPDATED_FIELD.getPreferredName(), settingsMetadataUpdated); + builder.field(TEMPLATES_METADATA_UPDATED_FIELD.getPreferredName(), templatesMetadataUpdated); + builder.startObject(INDICES_DIFF_FIELD.getPreferredName()); + builder.startArray(UPSERTS_FIELD.getPreferredName()); for (String index : indicesUpdated) { builder.value(index); } builder.endArray(); - builder.startArray("deletes"); + builder.startArray(DELETES_FIELD.getPreferredName()); for (String index : indicesDeleted) { builder.value(index); } builder.endArray(); builder.endObject(); - for (Map.Entry entry : customMetadataUpdated.entrySet()) { - if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); - } + // ToDo: add the custom metadata diff when we add a parser for this +// for (Map.Entry entry : customMetadataUpdated.entrySet()) { +// if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); +// } } builder.endObject(); - builder.field("cluster_blocks_diff", clusterBlocksUpdated); - builder.field("discovery_nodes_diff", discoveryNodesUpdated); + builder.field(CLUSTER_BLOCKS_UPDATED_FIELD.getPreferredName(), clusterBlocksUpdated); + builder.field(DISCOVERY_NODES_UPDATED_FIELD.getPreferredName(), discoveryNodesUpdated); } builder.endObject(); return builder; } + public static ClusterDiffManifest fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null, null); + } + public List findRemovedIndices(Map indices, Map previousIndices) { List removedIndices = new ArrayList<>(); for (String index : previousIndices.keySet()) { @@ -1121,5 +1214,88 @@ public boolean isClusterBlocksUpdated() { public boolean isDiscoveryNodesUpdated() { return discoveryNodesUpdated; } + + public static ClusterDiffManifest.Builder builder() { + return new Builder(); + } + + public static class Builder { + private String fromStateUUID; + private String toStateUUID; + private boolean coordinationMetadataUpdated; + private boolean settingsMetadataUpdated; + private boolean templatesMetadataUpdated; + private Map customMetadataUpdated; + private List indicesUpdated; + private List indicesDeleted; + private boolean clusterBlocksUpdated; + private boolean discoveryNodesUpdated; + public Builder() {} + + public Builder fromStateUUID(String fromStateUUID) { + this.fromStateUUID = fromStateUUID; + return this; + } + + public Builder toStateUUID(String toStateUUID) { + this.toStateUUID = toStateUUID; + return this; + } + + public Builder coordinationMetadataUpdated(boolean coordinationMetadataUpdated) { + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + return this; + } + + public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { + this.settingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + + public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { + this.templatesMetadataUpdated = templatesMetadataUpdated; + return this; + } + + public Builder customMetadataUpdated(Map customMetadataUpdated) { + this.customMetadataUpdated = customMetadataUpdated; + return this; + } + + public Builder indicesUpdated(List indicesUpdated) { + this.indicesUpdated = indicesUpdated; + return this; + } + + public Builder indicesDeleted(List indicesDeleted) { + this.indicesDeleted = indicesDeleted; + return this; + } + + public Builder clusterBlocksUpdated(boolean clusterBlocksUpdated) { + this.clusterBlocksUpdated = clusterBlocksUpdated; + return this; + } + + public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { + this.discoveryNodesUpdated = discoveryNodesUpdated; + return this; + } + + public ClusterDiffManifest build() { + return new ClusterDiffManifest( + fromStateUUID, + toStateUUID, + coordinationMetadataUpdated, + settingsMetadataUpdated, + templatesMetadataUpdated, + customMetadataUpdated, + indicesUpdated, + indicesDeleted, + clusterBlocksUpdated, + discoveryNodesUpdated + ); + } + } } } From 774f0c8cf72cdb6d962928adc6694530ca38b184 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Wed, 15 May 2024 17:00:30 +0530 Subject: [PATCH 051/133] Fixig build failures --- .../coordination/PublicationTransportHandler.java | 10 +++++----- .../gateway/remote/RemoteClusterStateUtils.java | 2 +- .../index/remote/RemoteIndexPathUploader.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index d65d730213ad4..99b0128f14c7d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -235,11 +235,6 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish } else if (manifest.getDiffManifest().getFromStateUUID().equals(lastSeen.stateUUID()) == false) { logger.debug("Last cluster state not compatible with the diff"); applyFullState = true; - } else { - ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get()); - final PublishWithJoinResponse response = acceptState(clusterState); - lastSeenClusterState.compareAndSet(lastSeen, clusterState); - return response; } if (applyFullState == true) { @@ -248,6 +243,11 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.set(clusterState); return response; + } else { + ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get()); + final PublishWithJoinResponse response = acceptState(clusterState); + lastSeenClusterState.compareAndSet(lastSeen, clusterState); + return response; } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 3fa9506192188..3263cabc50687 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -56,7 +56,7 @@ static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepositor /** * Exception for Remote state transfer. */ - static class RemoteStateTransferException extends RuntimeException { + public static class RemoteStateTransferException extends RuntimeException { public RemoteStateTransferException(String errorDesc) { super(errorDesc); diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index ff2114de54398..256cf8fbb4857 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -23,7 +23,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; import org.opensearch.gateway.remote.IndexMetadataUploadListener; -import org.opensearch.gateway.remote.RemoteClusterStateService.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; From 5febe86231ca0740aa074e3561cd0ef1b4994f00 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 16 May 2024 04:15:32 +0530 Subject: [PATCH 052/133] spotless changes for ClusterBlocks Signed-off-by: Shivansh Arora --- .../cluster/block/ClusterBlock.java | 10 +++- .../cluster/block/ClusterBlockTests.java | 44 +++++++++++---- .../cluster/block/ClusterBlocksTests.java | 55 ++++++++++++++----- 3 files changed, 82 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java index f048d1509ccdb..e131e4facc4b3 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java @@ -41,9 +41,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.xcontent.XContentParserUtils; import java.io.IOException; import java.util.EnumSet; @@ -64,7 +62,13 @@ public class ClusterBlock implements Writeable, ToXContentFragment { static final String KEY_RETRYABLE = "retryable"; static final String KEY_DISABLE_STATE_PERSISTENCE = "disable_state_persistence"; static final String KEY_LEVELS = "levels"; - private static final Set VALID_FIELDS = Sets.newHashSet(KEY_UUID, KEY_DESCRIPTION, KEY_RETRYABLE, KEY_DISABLE_STATE_PERSISTENCE, KEY_LEVELS); + private static final Set VALID_FIELDS = Sets.newHashSet( + KEY_UUID, + KEY_DESCRIPTION, + KEY_RETRYABLE, + KEY_DISABLE_STATE_PERSISTENCE, + KEY_LEVELS + ); private final int id; @Nullable private final String uuid; diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java index ec6234168030a..914a778e379c9 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java @@ -198,16 +198,40 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws } static String getExpectedXContentFragment(ClusterBlock clusterBlock, String indent) { - return indent + "\"" + clusterBlock.id() + "\" : {\n" - + (clusterBlock.uuid() != null ? - indent + " \"uuid\" : \""+ clusterBlock.uuid() + "\",\n" : "") - + indent + " \"description\" : \"" + clusterBlock.description() + "\",\n" - + indent + " \"retryable\" : " + clusterBlock.retryable() + ",\n" - + (clusterBlock.disableStatePersistence() ? - indent + " \"disable_state_persistence\" : " + clusterBlock.disableStatePersistence() + ",\n" : "") - + String.format(indent + " \"levels\" : [%s]\n", clusterBlock.levels().isEmpty() ? " " : - "\n" + String.join(",\n", clusterBlock.levels().stream().map(level -> indent + " \"" + level.name().toLowerCase(Locale.ROOT) + "\"").toArray(String[]::new)) + "\n " + indent) - + indent + "}"; + return indent + + "\"" + + clusterBlock.id() + + "\" : {\n" + + (clusterBlock.uuid() != null ? indent + " \"uuid\" : \"" + clusterBlock.uuid() + "\",\n" : "") + + indent + + " \"description\" : \"" + + clusterBlock.description() + + "\",\n" + + indent + + " \"retryable\" : " + + clusterBlock.retryable() + + ",\n" + + (clusterBlock.disableStatePersistence() + ? indent + " \"disable_state_persistence\" : " + clusterBlock.disableStatePersistence() + ",\n" + : "") + + String.format( + Locale.ROOT, + indent + " \"levels\" : [%s]\n", + clusterBlock.levels().isEmpty() + ? " " + : "\n" + + String.join( + ",\n", + clusterBlock.levels() + .stream() + .map(level -> indent + " \"" + level.name().toLowerCase(Locale.ROOT) + "\"") + .toArray(String[]::new) + ) + + "\n " + + indent + ) + + indent + + "}"; } static ClusterBlock randomClusterBlock() { diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java index c7710c3f397b3..86dda9eaa1a04 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -29,7 +30,6 @@ import static org.opensearch.cluster.block.ClusterBlockTests.getExpectedXContentFragment; import static org.opensearch.cluster.block.ClusterBlockTests.randomClusterBlock; - public class ClusterBlocksTests extends OpenSearchTestCase { public void testToXContent() throws IOException { ClusterBlocks clusterBlocks = randomClusterBlocks(); @@ -40,18 +40,45 @@ public void testToXContent() throws IOException { String expectedXContent = "{\n" + " \"blocks\" : {\n" - + String.format("%s", clusterBlocks.global().isEmpty() ? "" : - " \"global\" : {\n" - + clusterBlocks.global().stream().map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")).collect(Collectors.joining(",\n")) - + "\n }" + (!clusterBlocks.indices().isEmpty() ? "," : "") + "\n") - + String.format("%s", clusterBlocks.indices().isEmpty() ? "" : - " \"indices\" : {\n" - + clusterBlocks.indices().entrySet().stream().map(entry -> - " \"" + entry.getKey() + "\" : {" + (entry.getValue().isEmpty() ? " }" : "\n" - + entry.getValue().stream().map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")).collect(Collectors.joining(",\n")) - + "\n }") - ).collect(Collectors.joining(",\n")) - + "\n }\n") + + String.format( + Locale.ROOT, + "%s", + clusterBlocks.global().isEmpty() + ? "" + : " \"global\" : {\n" + + clusterBlocks.global() + .stream() + .map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")) + .collect(Collectors.joining(",\n")) + + "\n }" + + (!clusterBlocks.indices().isEmpty() ? "," : "") + + "\n" + ) + + String.format( + Locale.ROOT, + "%s", + clusterBlocks.indices().isEmpty() + ? "" + : " \"indices\" : {\n" + + clusterBlocks.indices() + .entrySet() + .stream() + .map( + entry -> " \"" + + entry.getKey() + + "\" : {" + + (entry.getValue().isEmpty() + ? " }" + : "\n" + + entry.getValue() + .stream() + .map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")) + .collect(Collectors.joining(",\n")) + + "\n }") + ) + .collect(Collectors.joining(",\n")) + + "\n }\n" + ) + " }\n" + "}"; @@ -87,7 +114,7 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws IllegalArgumentException.class, () -> ClusterBlocks.fromXContent(createParser(mediaType.xContent(), mutated)) ); - assertEquals("unknown field ["+ unsupportedField +"]", exception.getMessage()); + assertEquals("unknown field [" + unsupportedField + "]", exception.getMessage()); } else { try (XContentParser parser = createParser(JsonXContent.jsonXContent, originalBytes)) { ClusterBlocks parsedClusterBlocks = ClusterBlocks.fromXContent(parser); From bd45e85d5b13b9048e131a475cf98a0e7663de42 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 16 May 2024 12:36:54 +0530 Subject: [PATCH 053/133] remove duplicate diff object from manifest Signed-off-by: Shivansh Arora --- .../org/opensearch/gateway/remote/ClusterMetadataManifest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index f74eb128b059a..be6f66769c053 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -610,7 +610,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws attribute.toXContent(builder, params); } builder.endObject(); - diffManifest.toXContent(builder, params); } else if (onOrAfterCodecVersion(CODEC_V1)) { builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); From 54f1f3da14a47442d9973afe8694fe3243b37284 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 16 May 2024 12:21:37 +0530 Subject: [PATCH 054/133] Add remote-routing table diff Signed-off-by: Arpit Bandejiya --- .../remote/ClusterMetadataManifest.java | 98 ++++++++++++++++++- 1 file changed, 94 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index be6f66769c053..f44a8fb8ebb04 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -14,6 +14,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; @@ -1152,6 +1154,9 @@ public static class ClusterDiffManifest implements ToXContentObject { private static final ParseField DELETES_FIELD = new ParseField("deletes"); private static final ParseField CLUSTER_BLOCKS_UPDATED_FIELD = new ParseField("cluster_blocks_diff"); private static final ParseField DISCOVERY_NODES_UPDATED_FIELD = new ParseField("discovery_nodes_diff"); + private static final ParseField ROUTING_TABLE_DIFF = new ParseField("routing_table_diff"); + private static final ParseField ROUTING_TABLE_UPSERT_FIELD = new ParseField("routing_table_upsert"); + private static final ParseField ROUTING_TABLE_DELETE_FIELD = new ParseField("routing_table_delete"); private static final ObjectParser.NamedObjectParser PARSER; static { ConstructingObjectParser innerParser = new ConstructingObjectParser( @@ -1166,6 +1171,8 @@ public static class ClusterDiffManifest implements ToXContentObject { .indicesDeleted((List) fields[6]) .clusterBlocksUpdated((Boolean) fields[7]) .discoveryNodesUpdated((Boolean) fields[8]) + .indicesRoutingUpdated((List) fields[9]) + .indicesRoutingDeleted((List) fields[10]) .build() ); innerParser.declareString(ConstructingObjectParser.constructorArg(), FROM_STATE_UUID_FIELD); @@ -1177,6 +1184,9 @@ public static class ClusterDiffManifest implements ToXContentObject { innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), DELETES_FIELD); innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), DISCOVERY_NODES_UPDATED_FIELD); innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_BLOCKS_UPDATED_FIELD); + innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_UPSERT_FIELD); + innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_DELETE_FIELD); + PARSER = ((p, c, name) -> innerParser.parse(p, null)); } private final String fromStateUUID; @@ -1189,6 +1199,8 @@ public static class ClusterDiffManifest implements ToXContentObject { private final List indicesDeleted; private final boolean clusterBlocksUpdated; private final boolean discoveryNodesUpdated; + private final List indicesRoutingUpdated; + private final List indicesRoutingDeleted; ClusterDiffManifest(ClusterState state, ClusterState previousState) { fromStateUUID = previousState.stateUUID(); @@ -1207,9 +1219,22 @@ public static class ClusterDiffManifest implements ToXContentObject { state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) ); } - } - - public ClusterDiffManifest(String fromStateUUID, String toStateUUID, boolean coordinationMetadataUpdated, boolean settingsMetadataUpdated, boolean templatesMetadataUpdated, Map customMetadataUpdated, List indicesUpdated, List indicesDeleted, boolean clusterBlocksUpdated, boolean discoveryNodesUpdated) { + indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); + indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); + } + + public ClusterDiffManifest(String fromStateUUID, + String toStateUUID, + boolean coordinationMetadataUpdated, + boolean settingsMetadataUpdated, + boolean templatesMetadataUpdated, + Map customMetadataUpdated, + List indicesUpdated, + List indicesDeleted, + boolean clusterBlocksUpdated, + boolean discoveryNodesUpdated, + ListindicesRoutingUpdated, + ListindicesRoutingDeleted) { this.fromStateUUID = fromStateUUID; this.toStateUUID = toStateUUID; this.coordinationMetadataUpdated = coordinationMetadataUpdated; @@ -1220,6 +1245,8 @@ public ClusterDiffManifest(String fromStateUUID, String toStateUUID, boolean coo this.indicesDeleted = indicesDeleted; this.clusterBlocksUpdated = clusterBlocksUpdated; this.discoveryNodesUpdated = discoveryNodesUpdated; + this.indicesRoutingUpdated = indicesRoutingUpdated; + this.indicesRoutingDeleted = indicesRoutingDeleted; } @Override @@ -1252,6 +1279,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.field(CLUSTER_BLOCKS_UPDATED_FIELD.getPreferredName(), clusterBlocksUpdated); builder.field(DISCOVERY_NODES_UPDATED_FIELD.getPreferredName(), discoveryNodesUpdated); + + builder.startObject(ROUTING_TABLE_DIFF.getPreferredName()); + builder.startArray(ROUTING_TABLE_UPSERT_FIELD.getPreferredName()); + for (String index : indicesRoutingUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(ROUTING_TABLE_DELETE_FIELD.getPreferredName()); + for (String index : indicesRoutingDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); } return builder; } @@ -1283,6 +1323,34 @@ public List findUpdatedIndices(Map indices, Map getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List deletedIndices = new ArrayList<>(); + for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { + if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is deleted + deletedIndices.add(previousIndexRouting.getIndex().getName()); + } + } + return deletedIndices; + } + + public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List updatedIndicesRouting = new ArrayList<>(); + for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { + if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is created + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } else { + if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { + // if the latest routing table has the same routing table as the previous routing table, then the index is not updated + continue; + } + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } + } + return updatedIndicesRouting; + } + public String getFromStateUUID() { return fromStateUUID; } @@ -1323,6 +1391,14 @@ public boolean isDiscoveryNodesUpdated() { return discoveryNodesUpdated; } + public List getIndicesRoutingUpdated() { + return indicesRoutingUpdated; + } + + public List getIndicesRoutingDeleted() { + return indicesRoutingDeleted; + } + public static ClusterDiffManifest.Builder builder() { return new Builder(); } @@ -1338,6 +1414,8 @@ public static class Builder { private List indicesDeleted; private boolean clusterBlocksUpdated; private boolean discoveryNodesUpdated; + private List indicesRoutingUpdated; + private List indicesRoutingDeleted; public Builder() {} public Builder fromStateUUID(String fromStateUUID) { @@ -1390,6 +1468,16 @@ public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { return this; } + public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { + this.indicesRoutingUpdated = indicesRoutingUpdated; + return this; + } + + public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { + this.indicesRoutingDeleted = indicesRoutingDeleted; + return this; + } + public ClusterDiffManifest build() { return new ClusterDiffManifest( fromStateUUID, @@ -1401,7 +1489,9 @@ public ClusterDiffManifest build() { indicesUpdated, indicesDeleted, clusterBlocksUpdated, - discoveryNodesUpdated + discoveryNodesUpdated, + indicesRoutingUpdated, + indicesRoutingDeleted ); } } From f222634d1934ba61992900393f42361039bad1ad Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 16 May 2024 15:12:53 +0530 Subject: [PATCH 055/133] Add transport call to publish remote state --- .../coordination/CoordinationState.java | 4 ++ .../cluster/coordination/Coordinator.java | 2 +- .../PublicationTransportHandler.java | 58 ++++++++++++++++++- .../PublicationTransportHandlerTests.java | 2 +- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index 987a3e3ffa7d3..f1684bf94c506 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -104,6 +104,10 @@ public CoordinationState( this.isRemoteStateEnabled = isRemoteStoreClusterStateEnabled(settings); } + public boolean isRemoteStateEnabled() { + return isRemoteStateEnabled; + } + public long getCurrentTerm() { return persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL).getCurrentTerm(); } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 5ca2d93831506..1b3d6aeb0f8f5 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -1323,7 +1323,7 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) + clusterState; final PublicationTransportHandler.PublicationContext publicationContext = publicationHandler.newPublicationContext( - clusterChangedEvent + clusterChangedEvent, coordinationState.get().isRemoteStateEnabled() ); final PublishRequest publishRequest = coordinationState.get().handleClientValue(clusterState); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 99b0128f14c7d..252004f21b6de 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -77,6 +77,7 @@ public class PublicationTransportHandler { private static final Logger logger = LogManager.getLogger(PublicationTransportHandler.class); public static final String PUBLISH_STATE_ACTION_NAME = "internal:cluster/coordination/publish_state"; + public static final String PUBLISH_REMOTE_STATE_ACTION_NAME = "internal:cluster/coordination/publish_remote_state"; public static final String COMMIT_STATE_ACTION_NAME = "internal:cluster/coordination/commit_state"; private final TransportService transportService; @@ -123,6 +124,15 @@ public PublicationTransportHandler( (request, channel, task) -> channel.sendResponse(handleIncomingPublishRequest(request)) ); + transportService.registerRequestHandler( + PUBLISH_REMOTE_STATE_ACTION_NAME, + ThreadPool.Names.GENERIC, + false, + false, + RemotePublishRequest::new, + (request, channel, task) -> channel.sendResponse(handleIncomingRemotePublishRequest(request)) + ); + transportService.registerRequestHandler( COMMIT_STATE_ACTION_NAME, ThreadPool.Names.GENERIC, @@ -264,8 +274,8 @@ private PublishWithJoinResponse acceptState(ClusterState incomingState) { return handlePublishRequest.apply(new PublishRequest(incomingState)); } - public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent) { - final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent); + public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemoteStateEnabled) { + final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemoteStateEnabled); // Build the serializations we expect to need now, early in the process, so that an error during serialization fails the publication // straight away. This isn't watertight since we send diffs on a best-effort basis and may fall back to sending a full state (and @@ -310,12 +320,14 @@ public class PublicationContext { private final boolean sendFullVersion; private final Map serializedStates = new HashMap<>(); private final Map serializedDiffs = new HashMap<>(); + private final boolean sendRemoteState; - PublicationContext(ClusterChangedEvent clusterChangedEvent) { + PublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemoteStateEnabled) { discoveryNodes = clusterChangedEvent.state().nodes(); newState = clusterChangedEvent.state(); previousState = clusterChangedEvent.previousState(); sendFullVersion = previousState.getBlocks().disableStatePersistence(); + sendRemoteState = isRemoteStateEnabled; } void buildDiffAndSerializeStates() { @@ -379,6 +391,9 @@ public void onFailure(Exception e) { } else { responseActionListener = listener; } + if (sendRemoteState && destination.isRemoteStoreNode()) { + sendRemoteClusterState(destination, publishRequest.getAcceptedState(), responseActionListener); + } if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination); sendFullClusterState(destination, responseActionListener); @@ -424,6 +439,43 @@ public String executor() { ); } + private void sendRemoteClusterState(DiscoveryNode destination, ClusterState clusterState, ActionListener listener) { + try { + final RemotePublishRequest remotePublishRequest = new RemotePublishRequest(discoveryNodes.getLocalNode(), clusterState.term(), clusterState.getVersion(), clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()); + final Consumer transportExceptionHandler = exp -> { + logger.debug(() -> new ParameterizedMessage("failed to send remote cluster state to {}", destination), exp); + listener.onFailure(exp); + }; + final TransportResponseHandler responseHandler = new TransportResponseHandler< + PublishWithJoinResponse>() { + + @Override + public PublishWithJoinResponse read(StreamInput in) throws IOException { + return new PublishWithJoinResponse(in); + } + + @Override + public void handleResponse(PublishWithJoinResponse response) { + listener.onResponse(response); + } + + @Override + public void handleException(TransportException exp) { + transportExceptionHandler.accept(exp); + } + + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + }; + transportService.sendRequest(destination, PUBLISH_REMOTE_STATE_ACTION_NAME, remotePublishRequest, stateRequestOptions, responseHandler); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("error sending cluster state to {}", destination), e); + listener.onFailure(e); + } + } + private void sendFullClusterState(DiscoveryNode destination, ActionListener listener) { BytesReference bytes = serializedStates.get(destination.getVersion()); if (bytes == null) { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 94b9f8b71c362..27ebbf77fa28e 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -112,7 +112,7 @@ public void writeTo(StreamOutput out) throws IOException { OpenSearchException e = expectThrows( OpenSearchException.class, - () -> handler.newPublicationContext(new ClusterChangedEvent("test", unserializableClusterState, clusterState)) + () -> handler.newPublicationContext(new ClusterChangedEvent("test", unserializableClusterState, clusterState), false) ); assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IOException.class)); From 560ecb8714a29bdfff2deb6747ca285d05b4937a Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 16 May 2024 15:22:04 +0530 Subject: [PATCH 056/133] Fix EmptyMap warning in StreamReader Signed-off-by: Arpit Bandejiya --- .../remote/routingtable/IndexRoutingTableInputStreamReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java index 35ae9f287d7f2..847d387b228ca 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -44,7 +44,7 @@ public Map read() throws IOException { IndexRoutingTableHeader.read(in); int shards = in.readVInt(); logger.info("Number of Index Routing Table {}", shards); - Map indicesRouting = new HashMap(Collections.EMPTY_MAP); + Map indicesRouting = new HashMap<>(); for(int i=0; i Date: Thu, 16 May 2024 15:52:32 +0530 Subject: [PATCH 057/133] Add basic read-remote-routing flow Signed-off-by: Arpit Bandejiya --- .../cluster/routing/RoutingTable.java | 2 +- .../remote/RemoteRoutingTableService.java | 20 +++++++ .../IndexRoutingTableInputStream.java | 33 +++++------- .../IndexRoutingTableInputStreamReader.java | 35 +++++------- .../IndexRoutingTableInputStreamTests.java | 54 +++++++++++-------- 5 files changed, 80 insertions(+), 64 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java index e4095a84be081..6c7b94f316da2 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingTable.java @@ -79,7 +79,7 @@ public class RoutingTable implements Iterable, Diffable indicesRouting; - private RoutingTable(long version, final Map indicesRouting) { + public RoutingTable(long version, final Map indicesRouting) { this.version = version; this.indicesRouting = Collections.unmodifiableMap(indicesRouting); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 230178252b81f..017ac23ce1427 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -24,9 +24,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.index.Index; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStream; +import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStreamReader; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -40,6 +42,7 @@ import java.io.InputStream; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -159,6 +162,23 @@ public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState return null; } + public RoutingTable getLatestRoutingTable(long routingTableVersion, List indicesRoutingMetaData) throws IOException { + Map indicesRouting = new HashMap<>(); + + for(ClusterMetadataManifest.UploadedIndexMetadata indexRoutingMetaData: indicesRoutingMetaData) { + logger.debug("Starting the read for first indexRoutingMetaData: {}", indexRoutingMetaData); + String filePath = indexRoutingMetaData.getUploadedFilePath(); + BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(filePath)); + InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); + IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); + Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); + IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); + indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); + logger.debug("IndexRouting {}", indexRouting); + } + return new RoutingTable(routingTableVersion, indicesRouting); + } + private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) { } diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java index 3c249894a2ace..d4e2594bf153c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStream.java @@ -8,13 +8,10 @@ package org.opensearch.gateway.remote.routingtable; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamOutput; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import java.io.IOException; @@ -56,9 +53,7 @@ public class IndexRoutingTableInputStream extends InputStream { private static final int BUFFER_SIZE = 8192; private final IndexRoutingTableHeader indexRoutingTableHeader; - private final Iterator shardIter; - private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStream.class); private final BytesStreamOutput bytesStreamOutput; private final BufferedChecksumStreamOutput out; @@ -66,14 +61,12 @@ public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable) throws this(indexRoutingTable, BUFFER_SIZE); } - public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) - throws IOException { + public IndexRoutingTableInputStream(IndexRoutingTable indexRoutingTable, int size) throws IOException { this.buf = new byte[size]; this.shardIter = indexRoutingTable.iterator(); this.indexRoutingTableHeader = new IndexRoutingTableHeader(indexRoutingTable.getIndex().getName()); this.bytesStreamOutput = new BytesStreamOutput(); this.out = new BufferedChecksumStreamOutput(bytesStreamOutput); - logger.info("indexRoutingTable {}", indexRoutingTable.prettyPrint()); initialFill(indexRoutingTable.shards().size()); } @@ -91,7 +84,7 @@ private void initialFill(int shardCount) throws IOException { indexRoutingTableHeader.write(out); out.writeVInt(shardCount); - System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0 , buf, 0, bytesStreamOutput.bytes().length()); + System.arraycopy(bytesStreamOutput.bytes().toBytesRef().bytes, 0, buf, 0, bytesStreamOutput.bytes().length()); count = bytesStreamOutput.bytes().length(); bytesStreamOutput.reset(); fill(buf); @@ -99,17 +92,17 @@ private void initialFill(int shardCount) throws IOException { private void fill(byte[] buf) throws IOException { if (leftOverBuf != null) { - if(leftOverBuf.length > buf.length - count) { - // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in leftOverBuf. + if (leftOverBuf.length > buf.length - count) { + // leftOverBuf has more content than length of buf, so we need to copy only based on buf length and keep the remaining in + // leftOverBuf. System.arraycopy(leftOverBuf, 0, buf, count, buf.length - count); - byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; - System.arraycopy(leftOverBuf, buf.length - count , tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); + byte[] tempLeftOverBuffer = new byte[leftOverBuf.length - (buf.length - count)]; + System.arraycopy(leftOverBuf, buf.length - count, tempLeftOverBuffer, 0, leftOverBuf.length - (buf.length - count)); leftOverBuf = tempLeftOverBuffer; count = buf.length - count; - } else { System.arraycopy(leftOverBuf, 0, buf, count, leftOverBuf.length); - count += leftOverBuf.length; + count += leftOverBuf.length; leftOverBuf = null; } } @@ -117,8 +110,8 @@ private void fill(byte[] buf) throws IOException { if (count < buf.length && shardIter.hasNext()) { IndexShardRoutingTable next = shardIter.next(); IndexShardRoutingTable.Builder.writeTo(next, out); - //Add checksum for the file after all shards are done - if(!shardIter.hasNext()) { + // Add checksum for the file after all shards are done + if (!shardIter.hasNext()) { out.writeLong(out.getChecksum()); } out.flush(); @@ -132,12 +125,10 @@ private void fill(byte[] buf) throws IOException { } else { System.arraycopy(bytesRef.toBytesRef().bytes, 0, buf, count, buf.length - count); leftOverBuf = new byte[bytesRef.length() - (buf.length - count)]; - System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count , leftOverBuf, 0, bytesRef.length() - (buf.length - count)); + System.arraycopy(bytesRef.toBytesRef().bytes, buf.length - count, leftOverBuf, 0, bytesRef.length() - (buf.length - count)); count = buf.length; - } } - } private void maybeResizeAndFill() throws IOException { @@ -153,7 +144,7 @@ else if (pos >= buffer.length) { /* no room left in buffer */ markPos = -1; /* buffer got too big, invalidate mark */ pos = 0; /* drop buffer contents */ } else { /* grow buffer */ - int nsz = markLimit + 1; //NEED TO CHECK THIS + int nsz = markLimit + 1; byte[] nbuf = new byte[nsz]; System.arraycopy(buffer, 0, nbuf, 0, pos); buffer = nbuf; diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java index 847d387b228ca..e2b4f5da5f6e9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/IndexRoutingTableInputStreamReader.java @@ -13,19 +13,13 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.common.io.stream.BufferedChecksumStreamInput; -import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; -import java.io.BufferedReader; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; public class IndexRoutingTableInputStreamReader { @@ -34,32 +28,31 @@ public class IndexRoutingTableInputStreamReader { private static final Logger logger = LogManager.getLogger(IndexRoutingTableInputStreamReader.class); public IndexRoutingTableInputStreamReader(InputStream inputStream) throws IOException { - this.streamInput = new InputStreamStreamInput(inputStream); + streamInput = new InputStreamStreamInput(inputStream); } - public Map read() throws IOException { + public IndexRoutingTable readIndexRoutingTable(Index index) throws IOException { try { try (BufferedChecksumStreamInput in = new BufferedChecksumStreamInput(streamInput, "assertion")) { - // Read the Table Header first - IndexRoutingTableHeader.read(in); - int shards = in.readVInt(); - logger.info("Number of Index Routing Table {}", shards); - Map indicesRouting = new HashMap<>(); - for(int i=0; i indexShardRoutingTableMap = reader.read(); + IndexRoutingTable indexRoutingTable = reader.readIndexRoutingTable(metadata.index("test").getIndex()); - assertEquals(1, indexShardRoutingTableMap.size()); - assertNotNull(indexShardRoutingTableMap.get("test")); - assertEquals(2,indexShardRoutingTableMap.get("test").shards().size()); + assertEquals(1, indexRoutingTable.getShards().size()); + assertEquals(indexRoutingTable.getIndex(), metadata.index("test").getIndex()); + assertEquals(indexRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED).size(), 2); } catch (IOException e) { throw new RuntimeException(e); } }); } + public void testRoutingTableInputStreamWithInvalidIndex() { + Metadata metadata = Metadata.builder() + .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetadata.builder("invalid-index").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); + AtomicInteger assertionError = new AtomicInteger(); + initialRoutingTable.getIndicesRouting().values().forEach(indexShardRoutingTables -> { + try { + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexShardRoutingTables); + + IndexRoutingTableInputStreamReader reader = new IndexRoutingTableInputStreamReader(indexRoutingStream); + reader.readIndexRoutingTable(metadata.index("invalid-index").getIndex()); + + } catch (AssertionError e) { + assertionError.getAndIncrement(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + assertEquals(1, assertionError.get()); + } + } From 3186372de8860f6f5f4336e3f7ecebc87c87d190 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 17 May 2024 11:52:13 +0530 Subject: [PATCH 058/133] Move diff manifest to new class and fix fromXContent Signed-off-by: Shivansh Arora --- .../core/xcontent/XContentParserUtils.java | 12 + .../remote/ClusterMetadataManifest.java | 395 +--------------- .../remote/ClusterStateDiffManifest.java | 445 ++++++++++++++++++ .../remote/RemoteClusterStateService.java | 8 +- .../gateway/remote/RemoteManifestManager.java | 2 +- 5 files changed, 483 insertions(+), 379 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java diff --git a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java index b10be393f9adb..e0128a036148e 100644 --- a/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java +++ b/libs/core/src/main/java/org/opensearch/core/xcontent/XContentParserUtils.java @@ -38,6 +38,8 @@ import org.opensearch.core.xcontent.XContentParser.Token; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.function.Consumer; @@ -178,4 +180,14 @@ public static void parseTypedKeysObject(XContentParser parser, String delimi throw new ParsingException(parser.getTokenLocation(), "Failed to parse object: empty key"); } } + + public static List parseStringList(XContentParser parser) throws IOException { + List valueList = new ArrayList<>(); + ensureExpectedToken(Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != Token.END_ARRAY) { + ensureExpectedToken(Token.VALUE_STRING, parser.currentToken(), parser); + valueList.add(parser.text()); + } + return valueList; + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index f44a8fb8ebb04..9d80b53e79eaa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -11,27 +11,19 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentFragment; -import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; -import java.util.*; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -149,8 +141,8 @@ private static UploadedMetadataAttribute clusterBlocksMetadata(Object[] fields) return (UploadedMetadataAttribute) fields[16]; } - private static ClusterDiffManifest diffManifest(Object[] fields) { - return (ClusterDiffManifest) fields[17]; + private static ClusterStateDiffManifest diffManifest(Object[] fields) { + return (ClusterStateDiffManifest) fields[17]; } private static long routingTableVersion(Object[] fields) { @@ -319,7 +311,21 @@ private static void declareParser(ConstructingObjectParser= CODEC_V3) { - parser.declareNamedObject(ConstructingObjectParser.constructorArg(), ClusterDiffManifest.PARSER, DIFF_MANIFEST); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_DISCOVERY_NODES_METADATA + ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_CLUSTER_BLOCKS_METADATA + ); + parser.declareObject( + ConstructingObjectParser.constructorArg(), + (p, c) -> ClusterStateDiffManifest.fromXContent(p), + DIFF_MANIFEST + ); } if (codec_version >= CODEC_V4) { parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); @@ -349,7 +355,7 @@ private static void declareParser(ConstructingObjectParser indicesRouting; @@ -422,7 +428,7 @@ public UploadedMetadataAttribute getClusterBlocksMetadata() { return uploadedClusterBlocksMetadata; } - public ClusterDiffManifest getDiffManifest() { + public ClusterStateDiffManifest getDiffManifest() { return diffManifest; } @@ -463,7 +469,7 @@ public ClusterMetadataManifest( Map uploadedCustomMetadataMap, UploadedMetadataAttribute discoveryNodesMetadata, UploadedMetadataAttribute clusterBlocksMetadata, - ClusterDiffManifest diffManifest, + ClusterStateDiffManifest diffManifest, long routingTableVersion, List indicesRouting ) { @@ -748,7 +754,7 @@ public static class Builder { private String previousClusterUUID; private boolean committed; private boolean clusterUUIDCommitted; - private ClusterDiffManifest diffManifest; + private ClusterStateDiffManifest diffManifest; private long routingTableVersion; private List indicesRouting; @@ -865,7 +871,7 @@ public Builder clusterBlocksMetadata(UploadedMetadataAttribute clusterBlocksMeta return this; } - public Builder diffManifest(ClusterDiffManifest diffManifest) { + public Builder diffManifest(ClusterStateDiffManifest diffManifest) { this.diffManifest = diffManifest; return this; } @@ -1139,361 +1145,4 @@ public String toString() { + '}'; } } - - public static class ClusterDiffManifest implements ToXContentObject { - private static final ParseField FROM_STATE_UUID_FIELD = new ParseField("from_state_uuid"); - private static final ParseField TO_STATE_UUID_FIELD = new ParseField("to_state_uuid"); - private static final ParseField METADATA_DIFF_FIELD = new ParseField("metadata_diff"); - private static final ParseField COORDINATION_METADATA_UPDATED_FIELD = new ParseField( - "coordination_metadata_diff" - ); - private static final ParseField SETTINGS_METADATA_UPDATED_FIELD = new ParseField("settings_metadata_diff"); - private static final ParseField TEMPLATES_METADATA_UPDATED_FIELD = new ParseField("templates_metadata_diff"); - private static final ParseField INDICES_DIFF_FIELD = new ParseField("indices_diff"); - private static final ParseField UPSERTS_FIELD = new ParseField("upserts"); - private static final ParseField DELETES_FIELD = new ParseField("deletes"); - private static final ParseField CLUSTER_BLOCKS_UPDATED_FIELD = new ParseField("cluster_blocks_diff"); - private static final ParseField DISCOVERY_NODES_UPDATED_FIELD = new ParseField("discovery_nodes_diff"); - private static final ParseField ROUTING_TABLE_DIFF = new ParseField("routing_table_diff"); - private static final ParseField ROUTING_TABLE_UPSERT_FIELD = new ParseField("routing_table_upsert"); - private static final ParseField ROUTING_TABLE_DELETE_FIELD = new ParseField("routing_table_delete"); - private static final ObjectParser.NamedObjectParser PARSER; - static { - ConstructingObjectParser innerParser = new ConstructingObjectParser( - "cluster_diff_manifest", - fields -> ClusterDiffManifest.builder() - .fromStateUUID((String) fields[0]) - .toStateUUID((String) fields[1]) - .coordinationMetadataUpdated((Boolean) fields[2]) - .settingsMetadataUpdated((Boolean) fields[3]) - .templatesMetadataUpdated((Boolean) fields[4]) - .indicesUpdated((List) fields[5]) - .indicesDeleted((List) fields[6]) - .clusterBlocksUpdated((Boolean) fields[7]) - .discoveryNodesUpdated((Boolean) fields[8]) - .indicesRoutingUpdated((List) fields[9]) - .indicesRoutingDeleted((List) fields[10]) - .build() - ); - innerParser.declareString(ConstructingObjectParser.constructorArg(), FROM_STATE_UUID_FIELD); - innerParser.declareString(ConstructingObjectParser.constructorArg(), TO_STATE_UUID_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), COORDINATION_METADATA_UPDATED_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), SETTINGS_METADATA_UPDATED_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), TEMPLATES_METADATA_UPDATED_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), UPSERTS_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), DELETES_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), DISCOVERY_NODES_UPDATED_FIELD); - innerParser.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_BLOCKS_UPDATED_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_UPSERT_FIELD); - innerParser.declareStringArray(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_DELETE_FIELD); - - PARSER = ((p, c, name) -> innerParser.parse(p, null)); - } - private final String fromStateUUID; - private final String toStateUUID; - private final boolean coordinationMetadataUpdated; - private final boolean settingsMetadataUpdated; - private final boolean templatesMetadataUpdated; - private final Map customMetadataUpdated; - private final List indicesUpdated; - private final List indicesDeleted; - private final boolean clusterBlocksUpdated; - private final boolean discoveryNodesUpdated; - private final List indicesRoutingUpdated; - private final List indicesRoutingDeleted; - - ClusterDiffManifest(ClusterState state, ClusterState previousState) { - fromStateUUID = previousState.stateUUID(); - toStateUUID = state.stateUUID(); - coordinationMetadataUpdated = Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); - settingsMetadataUpdated = Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); - templatesMetadataUpdated = Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); - indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); - indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); - clusterBlocksUpdated = state.blocks().equals(previousState.blocks()); - discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); - customMetadataUpdated = new HashMap<>(); - for (String custom : state.metadata().customs().keySet()) { - customMetadataUpdated.put( - custom, - state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) - ); - } - indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); - indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); - } - - public ClusterDiffManifest(String fromStateUUID, - String toStateUUID, - boolean coordinationMetadataUpdated, - boolean settingsMetadataUpdated, - boolean templatesMetadataUpdated, - Map customMetadataUpdated, - List indicesUpdated, - List indicesDeleted, - boolean clusterBlocksUpdated, - boolean discoveryNodesUpdated, - ListindicesRoutingUpdated, - ListindicesRoutingDeleted) { - this.fromStateUUID = fromStateUUID; - this.toStateUUID = toStateUUID; - this.coordinationMetadataUpdated = coordinationMetadataUpdated; - this.settingsMetadataUpdated = settingsMetadataUpdated; - this.templatesMetadataUpdated = templatesMetadataUpdated; - this.customMetadataUpdated = customMetadataUpdated; - this.indicesUpdated = indicesUpdated; - this.indicesDeleted = indicesDeleted; - this.clusterBlocksUpdated = clusterBlocksUpdated; - this.discoveryNodesUpdated = discoveryNodesUpdated; - this.indicesRoutingUpdated = indicesRoutingUpdated; - this.indicesRoutingDeleted = indicesRoutingDeleted; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - { - builder.field(FROM_STATE_UUID_FIELD.getPreferredName(), fromStateUUID); - builder.field(TO_STATE_UUID_FIELD.getPreferredName(), toStateUUID); - builder.startObject(METADATA_DIFF_FIELD.getPreferredName()); - { - builder.field(COORDINATION_METADATA_UPDATED_FIELD.getPreferredName(), coordinationMetadataUpdated); - builder.field(SETTINGS_METADATA_UPDATED_FIELD.getPreferredName(), settingsMetadataUpdated); - builder.field(TEMPLATES_METADATA_UPDATED_FIELD.getPreferredName(), templatesMetadataUpdated); - builder.startObject(INDICES_DIFF_FIELD.getPreferredName()); - builder.startArray(UPSERTS_FIELD.getPreferredName()); - for (String index : indicesUpdated) { - builder.value(index); - } - builder.endArray(); - builder.startArray(DELETES_FIELD.getPreferredName()); - for (String index : indicesDeleted) { - builder.value(index); - } - builder.endArray(); - builder.endObject(); - // ToDo: add the custom metadata diff when we add a parser for this -// for (Map.Entry entry : customMetadataUpdated.entrySet()) { -// if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); -// } - } - builder.endObject(); - builder.field(CLUSTER_BLOCKS_UPDATED_FIELD.getPreferredName(), clusterBlocksUpdated); - builder.field(DISCOVERY_NODES_UPDATED_FIELD.getPreferredName(), discoveryNodesUpdated); - - builder.startObject(ROUTING_TABLE_DIFF.getPreferredName()); - builder.startArray(ROUTING_TABLE_UPSERT_FIELD.getPreferredName()); - for (String index : indicesRoutingUpdated) { - builder.value(index); - } - builder.endArray(); - builder.startArray(ROUTING_TABLE_DELETE_FIELD.getPreferredName()); - for (String index : indicesRoutingDeleted) { - builder.value(index); - } - builder.endArray(); - builder.endObject(); - } - return builder; - } - - public static ClusterDiffManifest fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null, null); - } - - public List findRemovedIndices(Map indices, Map previousIndices) { - List removedIndices = new ArrayList<>(); - for (String index : previousIndices.keySet()) { - // index present in previous state but not in current - if (!indices.containsKey(index)) { - removedIndices.add(index); - } - } - return removedIndices; - } - - public List findUpdatedIndices(Map indices, Map previousIndices) { - List updatedIndices = new ArrayList<>(); - for (String index : indices.keySet()) { - if (!previousIndices.containsKey(index)) { - updatedIndices.add(index); - } else if (previousIndices.get(index).getVersion() != indices.get(index).getVersion()) { - updatedIndices.add(index); - } - } - return updatedIndices; - } - - public List getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List deletedIndices = new ArrayList<>(); - for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { - if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is deleted - deletedIndices.add(previousIndexRouting.getIndex().getName()); - } - } - return deletedIndices; - } - - public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List updatedIndicesRouting = new ArrayList<>(); - for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { - if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is created - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } else { - if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { - // if the latest routing table has the same routing table as the previous routing table, then the index is not updated - continue; - } - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } - } - return updatedIndicesRouting; - } - - public String getFromStateUUID() { - return fromStateUUID; - } - - public String getToStateUUID() { - return toStateUUID; - } - - public boolean isCoordinationMetadataUpdated() { - return coordinationMetadataUpdated; - } - - public boolean isSettingsMetadataUpdated() { - return settingsMetadataUpdated; - } - - public boolean isTemplatesMetadataUpdated() { - return templatesMetadataUpdated; - } - - public Map getCustomMetadataUpdated() { - return customMetadataUpdated; - } - - public List getIndicesUpdated() { - return indicesUpdated; - } - - public List getIndicesDeleted() { - return indicesDeleted; - } - - public boolean isClusterBlocksUpdated() { - return clusterBlocksUpdated; - } - - public boolean isDiscoveryNodesUpdated() { - return discoveryNodesUpdated; - } - - public List getIndicesRoutingUpdated() { - return indicesRoutingUpdated; - } - - public List getIndicesRoutingDeleted() { - return indicesRoutingDeleted; - } - - public static ClusterDiffManifest.Builder builder() { - return new Builder(); - } - - public static class Builder { - private String fromStateUUID; - private String toStateUUID; - private boolean coordinationMetadataUpdated; - private boolean settingsMetadataUpdated; - private boolean templatesMetadataUpdated; - private Map customMetadataUpdated; - private List indicesUpdated; - private List indicesDeleted; - private boolean clusterBlocksUpdated; - private boolean discoveryNodesUpdated; - private List indicesRoutingUpdated; - private List indicesRoutingDeleted; - public Builder() {} - - public Builder fromStateUUID(String fromStateUUID) { - this.fromStateUUID = fromStateUUID; - return this; - } - - public Builder toStateUUID(String toStateUUID) { - this.toStateUUID = toStateUUID; - return this; - } - - public Builder coordinationMetadataUpdated(boolean coordinationMetadataUpdated) { - this.coordinationMetadataUpdated = coordinationMetadataUpdated; - return this; - } - - public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { - this.settingsMetadataUpdated = settingsMetadataUpdated; - return this; - } - - public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { - this.templatesMetadataUpdated = templatesMetadataUpdated; - return this; - } - - public Builder customMetadataUpdated(Map customMetadataUpdated) { - this.customMetadataUpdated = customMetadataUpdated; - return this; - } - - public Builder indicesUpdated(List indicesUpdated) { - this.indicesUpdated = indicesUpdated; - return this; - } - - public Builder indicesDeleted(List indicesDeleted) { - this.indicesDeleted = indicesDeleted; - return this; - } - - public Builder clusterBlocksUpdated(boolean clusterBlocksUpdated) { - this.clusterBlocksUpdated = clusterBlocksUpdated; - return this; - } - - public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { - this.discoveryNodesUpdated = discoveryNodesUpdated; - return this; - } - - public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { - this.indicesRoutingUpdated = indicesRoutingUpdated; - return this; - } - - public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { - this.indicesRoutingDeleted = indicesRoutingDeleted; - return this; - } - - public ClusterDiffManifest build() { - return new ClusterDiffManifest( - fromStateUUID, - toStateUUID, - coordinationMetadataUpdated, - settingsMetadataUpdated, - templatesMetadataUpdated, - customMetadataUpdated, - indicesUpdated, - indicesDeleted, - clusterBlocksUpdated, - discoveryNodesUpdated, - indicesRoutingUpdated, - indicesRoutingDeleted - ); - } - } - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java new file mode 100644 index 0000000000000..cc3603d01f78f --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -0,0 +1,445 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; + +public class ClusterStateDiffManifest implements ToXContentObject { + private static final String FROM_STATE_UUID_FIELD = "from_state_uuid"; + private static final String TO_STATE_UUID_FIELD = "to_state_uuid"; + private static final String METADATA_DIFF_FIELD = "metadata_diff"; + private static final String COORDINATION_METADATA_UPDATED_FIELD = "coordination_metadata_diff"; + private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff"; + private static final String TEMPLATES_METADATA_UPDATED_FIELD = ("templates_metadata_diff"); + private static final String INDICES_DIFF_FIELD = ("indices_diff"); + private static final String UPSERTS_FIELD = ("upserts"); + private static final String DELETES_FIELD = ("deletes"); + private static final String CLUSTER_BLOCKS_UPDATED_FIELD = ("cluster_blocks_diff"); + private static final String DISCOVERY_NODES_UPDATED_FIELD = ("discovery_nodes_diff"); + private static final String ROUTING_TABLE_DIFF = ("routing_table_diff"); + private static final String ROUTING_TABLE_UPSERT_FIELD = ("routing_table_upsert"); + private static final String ROUTING_TABLE_DELETE_FIELD = ("routing_table_delete"); + private final String fromStateUUID; + private final String toStateUUID; + private final boolean coordinationMetadataUpdated; + private final boolean settingsMetadataUpdated; + private final boolean templatesMetadataUpdated; + private final Map customMetadataUpdated; + private final List indicesUpdated; + private final List indicesDeleted; + private final boolean clusterBlocksUpdated; + private final boolean discoveryNodesUpdated; + private final List indicesRoutingUpdated; + private final List indicesRoutingDeleted; + + ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { + fromStateUUID = previousState.stateUUID(); + toStateUUID = state.stateUUID(); + coordinationMetadataUpdated = Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); + settingsMetadataUpdated = Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); + templatesMetadataUpdated = Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); + indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); + indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); + clusterBlocksUpdated = state.blocks().equals(previousState.blocks()); + discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); + customMetadataUpdated = new HashMap<>(); + for (String custom : state.metadata().customs().keySet()) { + customMetadataUpdated.put( + custom, + state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) + ); + } + indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); + indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); + } + + public ClusterStateDiffManifest(String fromStateUUID, + String toStateUUID, + boolean coordinationMetadataUpdated, + boolean settingsMetadataUpdated, + boolean templatesMetadataUpdated, + Map customMetadataUpdated, + List indicesUpdated, + List indicesDeleted, + boolean clusterBlocksUpdated, + boolean discoveryNodesUpdated, + ListindicesRoutingUpdated, + ListindicesRoutingDeleted) { + this.fromStateUUID = fromStateUUID; + this.toStateUUID = toStateUUID; + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + this.settingsMetadataUpdated = settingsMetadataUpdated; + this.templatesMetadataUpdated = templatesMetadataUpdated; + this.customMetadataUpdated = customMetadataUpdated; + this.indicesUpdated = indicesUpdated; + this.indicesDeleted = indicesDeleted; + this.clusterBlocksUpdated = clusterBlocksUpdated; + this.discoveryNodesUpdated = discoveryNodesUpdated; + this.indicesRoutingUpdated = indicesRoutingUpdated; + this.indicesRoutingDeleted = indicesRoutingDeleted; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + { + builder.field(FROM_STATE_UUID_FIELD, fromStateUUID); + builder.field(TO_STATE_UUID_FIELD, toStateUUID); + builder.startObject(METADATA_DIFF_FIELD); + { + builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); + builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); + builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); + builder.startObject(INDICES_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String index : indicesUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String index : indicesDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + // ToDo: add the custom metadata diff when we add a parser for this +// for (Map.Entry entry : customMetadataUpdated.entrySet()) { +// if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); +// } + } + builder.endObject(); + builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); + builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); + + builder.startObject(ROUTING_TABLE_DIFF); + builder.startArray(ROUTING_TABLE_UPSERT_FIELD); + for (String index : indicesRoutingUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(ROUTING_TABLE_DELETE_FIELD); + for (String index : indicesRoutingDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + } + return builder; + } + + public static ClusterStateDiffManifest fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(METADATA_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token.isValue()) { + switch (currentFieldName) { + case COORDINATION_METADATA_UPDATED_FIELD: + builder.coordinationMetadataUpdated(parser.booleanValue()); + break; + case SETTINGS_METADATA_UPDATED_FIELD: + builder.settingsMetadataUpdated(parser.booleanValue()); + break; + case TEMPLATES_METADATA_UPDATED_FIELD: + builder.templatesMetadataUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(INDICES_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.indicesUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.indicesDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + } else if (currentFieldName.equals(ROUTING_TABLE_DIFF)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case ROUTING_TABLE_UPSERT_FIELD: + builder.indicesRoutingUpdated(parseStringList(parser)); + break; + case ROUTING_TABLE_DELETE_FIELD: + builder.indicesRoutingDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token.isValue()) { + switch (currentFieldName) { + case FROM_STATE_UUID_FIELD: + builder.fromStateUUID(parser.text()); + break; + case TO_STATE_UUID_FIELD: + builder.toStateUUID(parser.text()); + break; + case CLUSTER_BLOCKS_UPDATED_FIELD: + builder.clusterBlocksUpdated(parser.booleanValue()); + break; + case DISCOVERY_NODES_UPDATED_FIELD: + builder.discoveryNodesUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + return builder.build(); + } + + public List findRemovedIndices(Map indices, Map previousIndices) { + List removedIndices = new ArrayList<>(); + for (String index : previousIndices.keySet()) { + // index present in previous state but not in current + if (!indices.containsKey(index)) { + removedIndices.add(index); + } + } + return removedIndices; + } + + public List findUpdatedIndices(Map indices, Map previousIndices) { + List updatedIndices = new ArrayList<>(); + for (String index : indices.keySet()) { + if (!previousIndices.containsKey(index)) { + updatedIndices.add(index); + } else if (previousIndices.get(index).getVersion() != indices.get(index).getVersion()) { + updatedIndices.add(index); + } + } + return updatedIndices; + } + + public List getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List deletedIndices = new ArrayList<>(); + for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { + if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is deleted + deletedIndices.add(previousIndexRouting.getIndex().getName()); + } + } + return deletedIndices; + } + + public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List updatedIndicesRouting = new ArrayList<>(); + for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { + if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is created + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } else { + if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { + // if the latest routing table has the same routing table as the previous routing table, then the index is not updated + continue; + } + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } + } + return updatedIndicesRouting; + } + + public String getFromStateUUID() { + return fromStateUUID; + } + + public String getToStateUUID() { + return toStateUUID; + } + + public boolean isCoordinationMetadataUpdated() { + return coordinationMetadataUpdated; + } + + public boolean isSettingsMetadataUpdated() { + return settingsMetadataUpdated; + } + + public boolean isTemplatesMetadataUpdated() { + return templatesMetadataUpdated; + } + + public Map getCustomMetadataUpdated() { + return customMetadataUpdated; + } + + public List getIndicesUpdated() { + return indicesUpdated; + } + + public List getIndicesDeleted() { + return indicesDeleted; + } + + public boolean isClusterBlocksUpdated() { + return clusterBlocksUpdated; + } + + public boolean isDiscoveryNodesUpdated() { + return discoveryNodesUpdated; + } + + public List getIndicesRoutingUpdated() { + return indicesRoutingUpdated; + } + + public List getIndicesRoutingDeleted() { + return indicesRoutingDeleted; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String fromStateUUID; + private String toStateUUID; + private boolean coordinationMetadataUpdated; + private boolean settingsMetadataUpdated; + private boolean templatesMetadataUpdated; + private Map customMetadataUpdated; + private List indicesUpdated; + private List indicesDeleted; + private boolean clusterBlocksUpdated; + private boolean discoveryNodesUpdated; + private List indicesRoutingUpdated; + private List indicesRoutingDeleted; + public Builder() {} + + public Builder fromStateUUID(String fromStateUUID) { + this.fromStateUUID = fromStateUUID; + return this; + } + + public Builder toStateUUID(String toStateUUID) { + this.toStateUUID = toStateUUID; + return this; + } + + public Builder coordinationMetadataUpdated(boolean coordinationMetadataUpdated) { + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + return this; + } + + public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { + this.settingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + + public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { + this.templatesMetadataUpdated = templatesMetadataUpdated; + return this; + } + + public Builder customMetadataUpdated(Map customMetadataUpdated) { + this.customMetadataUpdated = customMetadataUpdated; + return this; + } + + public Builder indicesUpdated(List indicesUpdated) { + this.indicesUpdated = indicesUpdated; + return this; + } + + public Builder indicesDeleted(List indicesDeleted) { + this.indicesDeleted = indicesDeleted; + return this; + } + + public Builder clusterBlocksUpdated(boolean clusterBlocksUpdated) { + this.clusterBlocksUpdated = clusterBlocksUpdated; + return this; + } + + public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { + this.discoveryNodesUpdated = discoveryNodesUpdated; + return this; + } + + public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { + this.indicesRoutingUpdated = indicesRoutingUpdated; + return this; + } + + public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { + this.indicesRoutingDeleted = indicesRoutingDeleted; + return this; + } + + public ClusterStateDiffManifest build() { + return new ClusterStateDiffManifest( + fromStateUUID, + toStateUUID, + coordinationMetadataUpdated, + settingsMetadataUpdated, + templatesMetadataUpdated, + customMetadataUpdated, + indicesUpdated, + indicesDeleted, + clusterBlocksUpdated, + discoveryNodesUpdated, + indicesRoutingUpdated, + indicesRoutingDeleted + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c5d17c225a9b2..f725a0cd345e5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,7 +18,6 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; -import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -33,7 +32,6 @@ import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; -import org.opensearch.gateway.remote.ClusterMetadataManifest.ClusterDiffManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -229,7 +227,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedCustomMetadataMap, uploadedMetadataResults.uploadedDiscoveryNodes, uploadedMetadataResults.uploadedClusterBlocks, - new ClusterMetadataManifest.ClusterDiffManifest(clusterState, ClusterState.EMPTY_STATE), + new ClusterStateDiffManifest(clusterState, ClusterState.EMPTY_STATE), routingIndexMetadata, false ); @@ -397,7 +395,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUpload || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), firstUpload || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), firstUpload || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), - new ClusterMetadataManifest.ClusterDiffManifest(clusterState, previousClusterState), + new ClusterStateDiffManifest(clusterState, previousClusterState), routingIndexMetadata, false ); this.latestClusterName = clusterState.getClusterName().value(); @@ -803,7 +801,7 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState) { assert manifest.getDiffManifest() != null; - ClusterDiffManifest diff = manifest.getDiffManifest(); + ClusterStateDiffManifest diff = manifest.getDiffManifest(); ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 83a4d7f98b284..c1e0ad4ece47a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -131,7 +131,7 @@ ClusterMetadataManifest uploadManifest( Map uploadedCustomMetadataMap, ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodesMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocksMetadata, - ClusterMetadataManifest.ClusterDiffManifest clusterDiffManifest, + ClusterStateDiffManifest clusterDiffManifest, List routingIndexMetadata, boolean committed ) throws IOException { synchronized (this) { From 8912b0c1b6432bc1c0abf66b28b0976e430ba336 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 17 May 2024 13:33:30 +0530 Subject: [PATCH 059/133] Fix cluster state publication --- .../coordination/CoordinationState.java | 1 + .../PublicationTransportHandler.java | 7 +++--- .../RemoteClusterStateAttributesManager.java | 24 ++++++++++++++++++ .../remote/RemoteClusterStateService.java | 25 +++++++++++++------ 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index f1684bf94c506..d8e48279a0db6 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -453,6 +453,7 @@ public PublishResponse handlePublishRequest(PublishRequest publishRequest) { clusterState.version(), clusterState.term() ); + logger.info("Setting last accepted state : term - {}, version - {}", clusterState.term(), clusterState.version()); persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL).setLastAcceptedState(clusterState); assert getLastAcceptedState() == clusterState; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 252004f21b6de..7c83707c5120a 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -248,13 +248,13 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish } if (applyFullState == true) { - ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest); + ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest, transportService.getLocalNode().getId()); logger.debug("Downloaded full cluster state version [{}]", clusterState.version()); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.set(clusterState); return response; } else { - ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get()); + ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get(), transportService.getLocalNode().getId()); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.compareAndSet(lastSeen, clusterState); return response; @@ -393,8 +393,7 @@ public void onFailure(Exception e) { } if (sendRemoteState && destination.isRemoteStoreNode()) { sendRemoteClusterState(destination, publishRequest.getAcceptedState(), responseActionListener); - } - if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { + } else if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination); sendFullClusterState(destination, responseActionListener); } else { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 1a1e7786256c0..d18db32c3f61b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -8,9 +8,11 @@ package org.opensearch.gateway.remote; +import java.util.Locale; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.blobstore.BlobContainer; @@ -76,6 +78,28 @@ CheckedRunnable getAsyncMetadataWriteAction( ); } + public ToXContent readMetadata(ChecksumBlobStoreFormat componentMetadataBlobStore, String clusterName, String clusterUUID, String fileName) { + final BlobContainer remoteStateAttributeContainer = clusterStateAttributeContainer(clusterName, clusterUUID); + try { + // Fetch custom metadata + if (fileName != null) { + String[] splitPath = fileName.split("/"); + return componentMetadataBlobStore.read( + remoteStateAttributeContainer, + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry() + ); + } else { + return TemplatesMetadata.EMPTY_METADATA; + } + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", fileName), + e + ); + } + } + private BlobContainer clusterStateAttributeContainer(String clusterName, String clusterUUID) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ return blobStoreRepository.blobStore() diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index f725a0cd345e5..21ecf2b56dd21 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; @@ -775,10 +776,10 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID ); } - return getClusterStateForManifest(clusterName, clusterMetadataManifest.get()); + return getClusterStateForManifest(clusterName, clusterMetadataManifest.get(), nodeId); } - public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest) { + public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId) { // todo make this async // Fetch Global Metadata Metadata globalMetadata = remoteGlobalMetadataManager.getGlobalMetadata(clusterName, manifest.getClusterUUID(), manifest); @@ -789,17 +790,20 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada manifest.getClusterUUID(), manifest ); + DiscoveryNodes discoveryNodes = (DiscoveryNodes) remoteClusterStateAttributesManager.readMetadata(DISCOVERY_NODES_FORMAT, clusterName, manifest.getClusterUUID(), manifest.getDiscoveryNodesMetadata().getUploadedFilename()); Map indexMetadataMap = new HashMap<>(); indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); }); return ClusterState.builder(ClusterState.EMPTY_STATE) .version(manifest.getStateVersion()) + .stateUUID(manifest.getStateUUID()) + .nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNodeId)) .metadata(Metadata.builder(globalMetadata).indices(indexMetadataMap).build()) .build(); } - public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState) { + public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) { assert manifest.getDiffManifest() != null; ClusterStateDiffManifest diff = manifest.getDiffManifest(); ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); @@ -827,11 +831,18 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata TemplatesMetadata templatesMetadata = remoteGlobalMetadataManager.getTemplatesMetadata(clusterName, manifest.getClusterUUID(), manifest.getTemplatesMetadata().getUploadedFilename()); metadataBuilder.templates(templatesMetadata); } - for (String customType : diff.getCustomMetadataUpdated().keySet()) { - Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(clusterName, manifest.getClusterUUID(), manifest.getCustomMetadataMap().get(customType).getUploadedFilename(), customType); - metadataBuilder.putCustom(customType, custom); + if (diff.isDiscoveryNodesUpdated()) { + DiscoveryNodes discoveryNodes = (DiscoveryNodes) remoteClusterStateAttributesManager.readMetadata(DISCOVERY_NODES_FORMAT, clusterName, manifest.getClusterUUID(), manifest.getDiscoveryNodesMetadata().getUploadedFilename()); + clusterStateBuilder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNodeId)); } - return clusterStateBuilder.metadata(metadataBuilder).build(); + if (diff.getCustomMetadataUpdated() != null) { + for (String customType : diff.getCustomMetadataUpdated().keySet()) { + Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(clusterName, manifest.getClusterUUID(), + manifest.getCustomMetadataMap().get(customType).getUploadedFilename(), customType); + metadataBuilder.putCustom(customType, custom); + } + } + return clusterStateBuilder.stateUUID(manifest.getStateUUID()).version(manifest.getStateVersion()).metadata(metadataBuilder).build(); } /** From b604b090ad5d5298644099b650d0fe67f51ca4f5 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 17 May 2024 10:43:02 +0530 Subject: [PATCH 060/133] Clean up global metadata attribute objects from remote Signed-off-by: Shivansh Arora --- .../RemoteClusterStateCleanupManager.java | 44 +++++++-- ...RemoteClusterStateCleanupManagerTests.java | 96 ++++++++++++++++++- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index ef40be2502199..b5568cae0572d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -148,7 +148,19 @@ void cleanUpStaleFiles() { } } - private void deleteClusterMetadata( + private void addStaleGlobalMetadataPath(String fileName, Set filesToKeep, Set staleGlobalMetadataPaths) { + if (!filesToKeep.contains(fileName)) { + String[] splitPath = fileName.split("/"); + staleGlobalMetadataPaths.add( + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( + splitPath[splitPath.length - 1] + ) + ); + } + } + + // visible for testing + void deleteClusterMetadata( String clusterName, String clusterUUID, List activeManifestBlobMetadata, @@ -167,7 +179,15 @@ private void deleteClusterMetadata( ); clusterMetadataManifest.getIndices() .forEach(uploadedIndexMetadata -> filesToKeep.add(uploadedIndexMetadata.getUploadedFilename())); - filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName()); + if (clusterMetadataManifest.getCodecVersion() == ClusterMetadataManifest.CODEC_V1) { + filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName()); + } else if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V2) { + filesToKeep.add(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); + filesToKeep.add(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); + filesToKeep.add(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); + clusterMetadataManifest.getCustomMetadataMap().values().forEach(attribute -> filesToKeep.add(attribute.getUploadedFilename())); + } + logger.info(filesToKeep); }); staleManifestBlobMetadata.forEach(blobMetadata -> { ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.fetchRemoteClusterMetadataManifest( @@ -176,14 +196,17 @@ private void deleteClusterMetadata( blobMetadata.name() ); staleManifestPaths.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blobMetadata.name()); - if (filesToKeep.contains(clusterMetadataManifest.getGlobalMetadataFileName()) == false) { - String[] globalMetadataSplitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); - staleGlobalMetadataPaths.add( - new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( - globalMetadataSplitPath[globalMetadataSplitPath.length - 1] - ) - ); + if (clusterMetadataManifest.getCodecVersion() == ClusterMetadataManifest.CODEC_V1) { + addStaleGlobalMetadataPath(clusterMetadataManifest.getGlobalMetadataFileName(), filesToKeep, staleGlobalMetadataPaths); + } else if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V2) { + addStaleGlobalMetadataPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename(), filesToKeep, staleGlobalMetadataPaths); + addStaleGlobalMetadataPath(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename(), filesToKeep, staleGlobalMetadataPaths); + addStaleGlobalMetadataPath(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename(), filesToKeep, staleGlobalMetadataPaths); + clusterMetadataManifest.getCustomMetadataMap().values().forEach(attribute -> { + addStaleGlobalMetadataPath(attribute.getUploadedFilename(), filesToKeep, staleGlobalMetadataPaths); + }); } + clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) { staleIndexMetadataPaths.add( @@ -297,7 +320,8 @@ public void onFailure(Exception e) { ); } - private void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { + // package private for testing + void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { logger.debug(String.format(Locale.ROOT, "Deleting stale files from remote - %s", stalePaths)); getBlobStoreTransferService().deleteBlobs( remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index b3e04abb6ad29..1a2d5c2ab6751 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -15,6 +15,7 @@ import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.support.PlainBlobMetadata; @@ -33,7 +34,10 @@ import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -42,15 +46,28 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; - +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.AsyncStaleFileDeletion; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES; import static org.opensearch.gateway.remote.RemoteClusterStateService.CLUSTER_STATE_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateService.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.encodeString; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; @@ -135,6 +152,81 @@ public void teardown() throws Exception { threadPool.shutdown(); } + public void testDeleteClusterMetadata() throws IOException { + String clusterUUID = "clusterUUID"; + String clusterName = "test-cluster"; + List inactiveBlobs = Arrays.asList( + new PlainBlobMetadata("manifest1.dat", 1L), + new PlainBlobMetadata("manifest2.dat", 1L), + new PlainBlobMetadata("manifest3.dat", 1L) + ); + List activeBlobs = Arrays.asList( + new PlainBlobMetadata("manifest4.dat", 1L), + new PlainBlobMetadata("manifest5.dat", 1L) + ); + UploadedIndexMetadata index1Metadata = new UploadedIndexMetadata("index1", "indexUUID1", "index_metadata1"); + UploadedIndexMetadata index2Metadata = new UploadedIndexMetadata("index2", "indexUUID2", "index_metadata2"); + UploadedIndexMetadata index1UpdatedMetadata = new UploadedIndexMetadata("index1", "indexUUID1", "index_metadata1_updated"); + UploadedMetadataAttribute coordinationMetadata = new UploadedMetadataAttribute(COORDINATION_METADATA, "coordination_metadata"); + UploadedMetadataAttribute templateMetadata = new UploadedMetadataAttribute(TEMPLATES_METADATA, "template_metadata"); + UploadedMetadataAttribute settingMetadata = new UploadedMetadataAttribute(SETTING_METADATA, "settings_metadata"); + UploadedMetadataAttribute coordinationMetadataUpdated = new UploadedMetadataAttribute(COORDINATION_METADATA, "coordination_metadata_updated"); + UploadedMetadataAttribute templateMetadataUpdated = new UploadedMetadataAttribute(TEMPLATES_METADATA, "template_metadata_updated"); + UploadedMetadataAttribute settingMetadataUpdated = new UploadedMetadataAttribute(SETTING_METADATA, "settings_metadata_updated"); + ClusterMetadataManifest manifest1 = ClusterMetadataManifest.builder() + .indices(List.of(index1Metadata)) + .globalMetadataFileName("global_metadata") + .clusterTerm(1L) + .stateVersion(1L) + .codecVersion(CODEC_V1) + .stateUUID(randomAlphaOfLength(10)) + .clusterUUID(clusterUUID) + .nodeId("nodeA") + .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) + .previousClusterUUID(ClusterState.UNKNOWN_UUID) + .committed(true) + .build(); + ClusterMetadataManifest manifest2 = ClusterMetadataManifest.builder(manifest1) + .indices(List.of(index1Metadata, index2Metadata)) + .codecVersion(CODEC_V2) + .globalMetadataFileName(null) + .coordinationMetadata(coordinationMetadata) + .templatesMetadata(templateMetadata) + .settingMetadata(settingMetadata) + .build(); + ClusterMetadataManifest manifest3 = ClusterMetadataManifest.builder(manifest2) + .indices(List.of(index1UpdatedMetadata, index2Metadata)) + .settingMetadata(settingMetadataUpdated) + .build(); + + // active manifest have reference to index1Updated, index2, settingsUpdated, coordinationUpdated, templates, templatesUpdated + ClusterMetadataManifest manifest4 = ClusterMetadataManifest.builder(manifest3) + .coordinationMetadata(coordinationMetadataUpdated) + .build(); + ClusterMetadataManifest manifest5 = ClusterMetadataManifest.builder(manifest4) + .templatesMetadata(templateMetadataUpdated) + .build(); + + when(remoteClusterStateService.fetchRemoteClusterMetadataManifest(eq(clusterName), eq(clusterUUID), any())) + .thenReturn(manifest4, manifest5, manifest1, manifest2, manifest3); + BlobContainer container = mock(BlobContainer.class); + when(blobStore.blobContainer(any())).thenReturn(container); + doNothing().when(container).deleteBlobsIgnoringIfNotExists(any()); + + remoteClusterStateCleanupManager.deleteClusterMetadata(clusterName, clusterUUID, activeBlobs, inactiveBlobs); + verify(container).deleteBlobsIgnoringIfNotExists( + List.of(new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + coordinationMetadata.getUploadedFilename() + ".dat", + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + settingMetadata.getUploadedFilename() + ".dat", + new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + "global_metadata.dat") + ); + verify(container).deleteBlobsIgnoringIfNotExists( + List.of(new BlobPath().add(INDEX_PATH_TOKEN).add(index1Metadata.getIndexUUID()).buildAsString() + index1Metadata.getUploadedFilePath() + ".dat") + ); + Set staleManifest = new HashSet<>(); + inactiveBlobs.forEach(blob -> staleManifest.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blob.name())); + verify(container).deleteBlobsIgnoringIfNotExists(new ArrayList<>(staleManifest)); + } + public void testDeleteStaleClusterUUIDs() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); ClusterMetadataManifest clusterMetadataManifest = ClusterMetadataManifest.builder() @@ -322,7 +414,7 @@ public void testRemoteCleanupCallsDeleteIfVersionIncrementGreaterThanThreshold() assertEquals(1, callCount.get()); } - public void testRemoteCleanupSchedulesEvenAfterFailure() throws InterruptedException { + public void testRemoteCleanupSchedulesEvenAfterFailure() { remoteClusterStateCleanupManager.start(); RemoteClusterStateCleanupManager spyManager = spy(remoteClusterStateCleanupManager); AtomicInteger callCount = new AtomicInteger(0); From dc204c1e18c3b51f3619c891a06f631657c72d00 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Tue, 14 May 2024 14:36:43 +0530 Subject: [PATCH 061/133] Refactor remote state --- .../remote/AbstractRemoteBlobStoreObject.java | 24 ++++ .../gateway/remote/BlobPathParameters.java | 32 +++++ .../ChecksumBlobStoreFormatRegistry.java | 18 +++ .../gateway/remote/RemoteIndexMetadata.java | 114 ++++++++++++++++++ .../gateway/remote/RemoteObject.java | 22 ++++ .../gateway/remote/RemoteObjectBlobStore.java | 88 ++++++++++++++ .../gateway/remote/RemoteObjectManager.java | 39 ++++++ .../gateway/remote/RemoteObjectStore.java | 26 ++++ .../gateway/remote/RemoteStoreFactory.java | 19 +++ .../blobstore/ChecksumBlobStoreFormat.java | 11 ++ 10 files changed, 393 insertions(+) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java new file mode 100644 index 0000000000000..f6e3df100e5d1 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public abstract class AbstractRemoteBlobStoreObject implements RemoteObject { + + public abstract BlobContainer getBlobContainer(); + public abstract BlobPathParameters getBlobPathParameters(); + public abstract String getBlobNameFormat(); + public abstract String getBlobName(); + public abstract String generateBlobFileName(); + public abstract RemoteObjectStore getBackingStore(); + public abstract ChecksumBlobStoreFormat getChecksumBlobStoreFormat(); +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java b/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java new file mode 100644 index 0000000000000..e6d101c27584a --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import java.util.List; + +public class BlobPathParameters { + + public static final String FILE_NAME_DELIMITER = "__"; + + private List pathTokens; + private String filePrefix; + + public BlobPathParameters(List pathTokens, String filePrefix) { + this.pathTokens = pathTokens; + this.filePrefix = filePrefix; + } + + public List getPathTokens() { + return pathTokens; + } + + public String getFilePrefix() { + return filePrefix; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java b/server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java new file mode 100644 index 0000000000000..5708367f15b56 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import java.util.Map; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class ChecksumBlobStoreFormatRegistry { + + public static Map, ChecksumBlobStoreFormat> checksumBlobStoreFormatMap; + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java new file mode 100644 index 0000000000000..32f009e931052 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class RemoteIndexMetadata extends AbstractRemoteBlobStoreObject { + + public static final String DELIMITER = "__"; + public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; + + public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "index-metadata", + RemoteClusterStateUtils.METADATA_NAME_FORMAT, + IndexMetadata::fromXContent + ); + + public static final ToXContent.Params FORMAT_PARAMS; + + static { + Map params = new HashMap<>(1); + params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY); + FORMAT_PARAMS = new ToXContent.MapParams(params); + } + + private IndexMetadata indexMetadata; + private final RemoteObjectStore backingStore; + private String blobName; + + public RemoteIndexMetadata(IndexMetadata indexMetadata, RemoteObjectStore backingStore) { + this.indexMetadata = indexMetadata; + this.backingStore = backingStore; + } + + public RemoteIndexMetadata(String blobName, RemoteObjectStore backingStore) { + this.blobName = blobName; + this.backingStore = backingStore; + } + + @Override + public IndexMetadata get() { + return indexMetadata; + } + + @Override + public BlobContainer getBlobContainer() { + return null; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("index"), "metadata"); + } + + @Override + public ChecksumBlobStoreFormat getChecksumBlobStoreFormat() { + return INDEX_METADATA_FORMAT; + } + + @Override + public String getBlobNameFormat() { + return null; + } + + @Override + public String getBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + return String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(indexMetadata.getVersion()), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last + // place to determine codec version. + ); + } + + @Override + public RemoteObjectStore getBackingStore() { + return backingStore; + } + + @Override + public BytesReference serialize() throws IOException { + return INDEX_METADATA_FORMAT.serialize(indexMetadata, generateBlobFileName(), getBackingStore().getCompressor(), FORMAT_PARAMS); + } + + @Override + public IndexMetadata deserialize(InputStream inputStream) throws IOException { + return null; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java new file mode 100644 index 0000000000000..46b706583809b --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import java.io.IOException; +import java.io.InputStream; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.core.common.bytes.BytesReference; + +public interface RemoteObject { + public T get(); + public RemoteObjectStore getBackingStore(); + public BytesReference serialize() throws IOException; + + T deserialize(InputStream inputStream) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java new file mode 100644 index 0000000000000..9a84243ae0006 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.compress.Compressor; +import org.opensearch.repositories.blobstore.BlobStoreRepository; + +public class RemoteObjectBlobStore implements RemoteObjectStore { + private final BlobStoreRepository blobStoreRepository; + private final String clusterName; + private final String clusterUUID; + + public RemoteObjectBlobStore(BlobStoreRepository remoteStoreFactory, String clusterName, String clusterUUID) { + this.blobStoreRepository = remoteStoreFactory; + this.clusterName = clusterName; + this.clusterUUID = clusterUUID; + } + + @Override + public void write(RemoteObject remoteObject) throws IOException { + AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; + BytesReference bytesReference = abstractRemoteBlobStoreObject.serialize(); + abstractRemoteBlobStoreObject.getBlobContainer().writeBlob(abstractRemoteBlobStoreObject.generateBlobFileName(), bytesReference.streamInput(), bytesReference.length(), false); + } + + @Override + public CheckedRunnable writeAsync(RemoteObject remoteObject, ActionListener listener) { + return () -> { + AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; + BytesReference bytesReference = abstractRemoteBlobStoreObject.serialize(); + abstractRemoteBlobStoreObject.getBlobContainer().writeBlob(abstractRemoteBlobStoreObject.generateBlobFileName(), bytesReference.streamInput(), bytesReference.length(), false); + }; + } + + @Override + public T read(RemoteObject remoteObject) throws IOException { + AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; + return abstractRemoteBlobStoreObject.deserialize(getBlobContainer(abstractRemoteBlobStoreObject.getBlobPathParameters().getPathTokens()).readBlob(abstractRemoteBlobStoreObject.getBlobName())); + } + + @Override + public CompletableFuture readAsync(RemoteObject remoteObject){ + return CompletableFuture.supplyAsync(() -> { + AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; + try { + return abstractRemoteBlobStoreObject.deserialize(getBlobContainer(abstractRemoteBlobStoreObject.getBlobPathParameters().getPathTokens()).readBlob(abstractRemoteBlobStoreObject.getBlobName())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @Override + public Compressor getCompressor() { + return blobStoreRepository.getCompressor(); + } + + private BlobContainer getBlobContainer(List pathTokens) { + BlobPath blobPath = blobStoreRepository.basePath().add(encodeString(clusterName)).add("cluster-state").add(clusterUUID); + for (String token : pathTokens) { + blobPath = blobPath.add(token); + } + return blobStoreRepository.blobStore().blobContainer(blobPath); + } + + public static String encodeString(String content) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8)); + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java new file mode 100644 index 0000000000000..acacaf1ada6da --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.core.xcontent.ToXContent; + +public class RemoteObjectManager { + + private final RemoteStoreFactory remoteStoreFactory; + private final String clusterName; + private final String clusterUUID; + + public RemoteObjectManager(RemoteStoreFactory remoteStoreFactory, String clusterName, String clusterUUID) { + this.remoteStoreFactory = remoteStoreFactory; + this.clusterName = clusterName; + this.clusterUUID = clusterUUID; + } + + public void write(RemoteObject remoteObject) throws IOException { + remoteObject.getBackingStore().write(remoteObject); + } + + public T read(RemoteObject remoteObject) throws IOException { + return remoteObject.getBackingStore().read(remoteObject); + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java new file mode 100644 index 0000000000000..451fbe121d48e --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.ToXContent; + +public interface RemoteObjectStore { + + public void write(RemoteObject remoteObject) throws IOException; + public CheckedRunnable writeAsync(RemoteObject remoteObject, ActionListener listener); + public T read(RemoteObject remoteObject) throws IOException; + public CompletableFuture readAsync(RemoteObject remoteObject); + public Compressor getCompressor(); + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java new file mode 100644 index 0000000000000..72714bc8baba5 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.repositories.blobstore.BlobStoreRepository; + +public class RemoteStoreFactory { + + public BlobStoreRepository getBlobStoreRepository() { + return null; + } + +} diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java index e567e1b626c5a..b5812f760ede2 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -31,6 +31,7 @@ package org.opensearch.repositories.blobstore; +import java.util.concurrent.CompletableFuture; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -119,6 +120,16 @@ public T read(BlobContainer blobContainer, String name, NamedXContentRegistry na return deserialize(blobName, namedXContentRegistry, Streams.readFully(blobContainer.readBlob(blobName))); } + public CompletableFuture readAsync(BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry) throws IOException { + return CompletableFuture.supplyAsync(() -> { + try { + return ChecksumBlobStoreFormat.this.read(blobContainer, name, namedXContentRegistry); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + public String blobName(String name) { return String.format(Locale.ROOT, getBlobNameFormat(), name); } From 405687795744f3d921d71e8a8b44539cc4c46673 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Sat, 18 May 2024 20:38:06 +0530 Subject: [PATCH 062/133] Add Code for reading from diff file and constructing the Routing Table Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 017ac23ce1427..eda1878896cff 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.common.CheckedRunnable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -162,6 +163,31 @@ public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState return null; } + public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest manifest) throws IOException { + List indicesRoutingDeleted = manifest.getDiffManifest().getIndicesRoutingDeleted(); + List indicesRoutingUpdated = manifest.getDiffManifest().getIndicesRoutingUpdated(); + + List indicesRoutingUpdatedMetadata = manifest.getIndicesRouting().stream() + .filter(indexRouting -> indicesRoutingUpdated.contains(indexRouting.getIndexName())) + .collect(Collectors.toList()); + + Map indicesRouting = previousClusterState.getRoutingTable().indicesRouting(); + indicesRoutingDeleted.forEach(indicesRouting::remove); + + for(ClusterMetadataManifest.UploadedIndexMetadata indexRoutingMetaData: indicesRoutingUpdatedMetadata) { + logger.debug("Starting the read for first indexRoutingMetaData: {}", indexRoutingMetaData); + String filePath = indexRoutingMetaData.getUploadedFilePath(); + BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(filePath)); + InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); + IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); + Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); + IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); + indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); + logger.debug("IndexRouting {}", indexRouting); + } + return new RoutingTable(manifest.getRoutingTableVersion(), indicesRouting); + } + public RoutingTable getLatestRoutingTable(long routingTableVersion, List indicesRoutingMetaData) throws IOException { Map indicesRouting = new HashMap<>(); From 70fae2952dd46de38aee26ff858d2864f4b41ada Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Sun, 19 May 2024 00:47:29 +0530 Subject: [PATCH 063/133] Add Remote Routing read in remote cluster state publication Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 39 ++++++++++++------- .../remote/ClusterStateDiffManifest.java | 20 +++++----- .../remote/RemoteClusterStateService.java | 34 +++++++++++++++- 3 files changed, 67 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index eda1878896cff..9a6f746ea8003 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -163,7 +163,7 @@ public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState return null; } - public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest manifest) throws IOException { + public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest manifest){ List indicesRoutingDeleted = manifest.getDiffManifest().getIndicesRoutingDeleted(); List indicesRoutingUpdated = manifest.getDiffManifest().getIndicesRoutingUpdated(); @@ -178,29 +178,40 @@ public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState logger.debug("Starting the read for first indexRoutingMetaData: {}", indexRoutingMetaData); String filePath = indexRoutingMetaData.getUploadedFilePath(); BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(filePath)); - InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); - IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); - Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); - IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); - indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); - logger.debug("IndexRouting {}", indexRouting); + try { + InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); + IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); + Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); + IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); + indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); + logger.debug("IndexRouting {}", indexRouting); + } catch (IOException e) { + logger.info("RoutingTable read failed with error: {}", e.toString()); + } + } return new RoutingTable(manifest.getRoutingTableVersion(), indicesRouting); } - public RoutingTable getLatestRoutingTable(long routingTableVersion, List indicesRoutingMetaData) throws IOException { + public RoutingTable getFullRoutingTable(long routingTableVersion, List indicesRoutingMetaData) { Map indicesRouting = new HashMap<>(); for(ClusterMetadataManifest.UploadedIndexMetadata indexRoutingMetaData: indicesRoutingMetaData) { logger.debug("Starting the read for first indexRoutingMetaData: {}", indexRoutingMetaData); String filePath = indexRoutingMetaData.getUploadedFilePath(); BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(filePath)); - InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); - IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); - Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); - IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); - indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); - logger.debug("IndexRouting {}", indexRouting); + + try { + InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); + IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); + Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); + IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); + indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); + logger.debug("IndexRouting {}", indexRouting); + } catch (IOException e) { + logger.info("RoutingTable read failed with error: {}", e.toString()); + } + } return new RoutingTable(routingTableVersion, indicesRouting); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index cc3603d01f78f..c184166c490c3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -40,8 +40,6 @@ public class ClusterStateDiffManifest implements ToXContentObject { private static final String CLUSTER_BLOCKS_UPDATED_FIELD = ("cluster_blocks_diff"); private static final String DISCOVERY_NODES_UPDATED_FIELD = ("discovery_nodes_diff"); private static final String ROUTING_TABLE_DIFF = ("routing_table_diff"); - private static final String ROUTING_TABLE_UPSERT_FIELD = ("routing_table_upsert"); - private static final String ROUTING_TABLE_DELETE_FIELD = ("routing_table_delete"); private final String fromStateUUID; private final String toStateUUID; private final boolean coordinationMetadataUpdated; @@ -134,12 +132,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); builder.startObject(ROUTING_TABLE_DIFF); - builder.startArray(ROUTING_TABLE_UPSERT_FIELD); + builder.startArray(UPSERTS_FIELD); for (String index : indicesRoutingUpdated) { builder.value(index); } builder.endArray(); - builder.startArray(ROUTING_TABLE_DELETE_FIELD); + builder.startArray(DELETES_FIELD); for (String index : indicesRoutingDeleted) { builder.value(index); } @@ -206,14 +204,14 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw } } } else if (currentFieldName.equals(ROUTING_TABLE_DIFF)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { currentFieldName = parser.currentName(); - token = parser.nextToken(); + parser.nextToken(); switch (currentFieldName) { - case ROUTING_TABLE_UPSERT_FIELD: + case UPSERTS_FIELD: builder.indicesRoutingUpdated(parseStringList(parser)); break; - case ROUTING_TABLE_DELETE_FIELD: + case DELETES_FIELD: builder.indicesRoutingDeleted(parseStringList(parser)); break; default: @@ -271,14 +269,14 @@ public List findUpdatedIndices(Map indices, Map getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List deletedIndices = new ArrayList<>(); + List deletedIndicesRouting = new ArrayList<>(); for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { // Latest Routing Table does not have entry for the index which means the index is deleted - deletedIndices.add(previousIndexRouting.getIndex().getName()); + deletedIndicesRouting.add(previousIndexRouting.getIndex().getName()); } } - return deletedIndices; + return deletedIndicesRouting; } public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 21ecf2b56dd21..c02154d10c004 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,6 +18,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.CheckedRunnable; @@ -171,6 +172,8 @@ public RemoteClusterStateService( this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, settings, clusterSettings); logger.info("REMOTE ROUTING ENABLED"); + } else { + logger.info("REMOTE ROUTING DISABLED"); } this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, this::updateCleanupInterval); @@ -795,6 +798,18 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada Map indexMetadataMap = new HashMap<>(); indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); }); + if(remoteRoutingTableService != null) { + logger.info(" The Remote Routing Table is fetching the Full Routing Table"); + RoutingTable routingTable = remoteRoutingTableService.getFullRoutingTable(manifest.getRoutingTableVersion(), manifest.getIndicesRouting()); + return ClusterState.builder(ClusterState.EMPTY_STATE) + .version(manifest.getStateVersion()) + .stateUUID(manifest.getStateUUID()) + .nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNodeId)) + .metadata(Metadata.builder(globalMetadata).indices(indexMetadataMap).build()) + .routingTable(routingTable) + .build(); + } + return ClusterState.builder(ClusterState.EMPTY_STATE) .version(manifest.getStateVersion()) .stateUUID(manifest.getStateUUID()) @@ -842,7 +857,24 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata metadataBuilder.putCustom(customType, custom); } } - return clusterStateBuilder.stateUUID(manifest.getStateUUID()).version(manifest.getStateVersion()).metadata(metadataBuilder).build(); + + // Constructing Routing Table + if(remoteRoutingTableService!= null){ + RoutingTable routingTable = remoteRoutingTableService.getIncrementalRoutingTable(previousState, manifest); + return clusterStateBuilder. + stateUUID(manifest.getStateUUID()). + version(manifest.getStateVersion()). + metadata(metadataBuilder). + routingTable(routingTable). + build(); + } + + + return clusterStateBuilder. + stateUUID(manifest.getStateUUID()). + version(manifest.getStateVersion()). + metadata(metadataBuilder). + build(); } /** From 093d92605ca0cb945a38b35a3452a0c1cfea7f36 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sun, 19 May 2024 12:36:47 +0530 Subject: [PATCH 064/133] Add clusterUUID when downloading state --- .../gateway/remote/RemoteGlobalMetadataManager.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 12fb618152639..4f650e3bf2954 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -160,6 +160,10 @@ Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetada builder.coordinationMetadata(coordinationMetadata); builder.persistentSettings(settingsMetadata); builder.templates(templatesMetadata); + builder.clusterUUID(clusterMetadataManifest.getClusterUUID()); + builder.clusterUUIDCommitted(clusterMetadataManifest.isClusterUUIDCommitted()); + //todo add metadata version + //builder.version(clusterMetadataManifest.met) clusterMetadataManifest.getCustomMetadataMap() .forEach( (key, value) -> builder.putCustom( From 034f54b9bacc83241de84b94335d94c92a4e0bd8 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Mon, 20 May 2024 16:16:40 +0530 Subject: [PATCH 065/133] Add fixes for remote routing table Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 9a6f746ea8003..8a66e3ece8253 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -96,7 +96,7 @@ public List writeFullRoutingTable BlobPath custerMetadataBasePath = getCusterMetadataBasePath(blobStoreRepository, clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()); for (IndexRoutingTable indexRouting : currentRoutingTable.getIndicesRouting().values()) { - uploadedIndices.add(uploadIndex(indexRouting, clusterState.getRoutingTable().version(), custerMetadataBasePath)); + uploadedIndices.add(uploadIndex(indexRouting, custerMetadataBasePath)); } logger.info("uploadedIndices {}", uploadedIndices); @@ -124,19 +124,19 @@ public List writeIncrementalRouti uploadedIndices.add(allUploadedIndicesRouting.get(indexRouting.getIndex().getName())); } else { // new index or shards changed, in both cases we upload new index file. - uploadedIndices.add(uploadIndex(indexRouting, clusterState.getRoutingTable().version(), custerMetadataBasePath)); + uploadedIndices.add(uploadIndex(indexRouting, custerMetadataBasePath)); } } return uploadedIndices; } - private ClusterMetadataManifest.UploadedIndexMetadata uploadIndex(IndexRoutingTable indexRouting, long routingTableVersion, BlobPath custerMetadataBasePath) { + private ClusterMetadataManifest.UploadedIndexMetadata uploadIndex(IndexRoutingTable indexRouting, BlobPath custerMetadataBasePath) { try { InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting); BlobContainer container = blobStoreRepository.blobStore().blobContainer(custerMetadataBasePath.add(INDEX_ROUTING_PATH_TOKEN).add(indexRouting.getIndex().getUUID())); - - container.writeBlob(getIndexRoutingFileName(), indexRoutingStream, 4096, true); - return new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString() + getIndexRoutingFileName()); + String indexRoutingFileName = getIndexRoutingFileName(); + container.writeBlob(indexRoutingFileName, indexRoutingStream, 4096, true); + return new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString() + indexRoutingFileName); } catch (IOException e) { logger.error("Failed to write {}", e); @@ -171,15 +171,15 @@ public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState .filter(indexRouting -> indicesRoutingUpdated.contains(indexRouting.getIndexName())) .collect(Collectors.toList()); - Map indicesRouting = previousClusterState.getRoutingTable().indicesRouting(); + Map indicesRouting = new HashMap<>(previousClusterState.getRoutingTable().indicesRouting()); indicesRoutingDeleted.forEach(indicesRouting::remove); for(ClusterMetadataManifest.UploadedIndexMetadata indexRoutingMetaData: indicesRoutingUpdatedMetadata) { logger.debug("Starting the read for first indexRoutingMetaData: {}", indexRoutingMetaData); String filePath = indexRoutingMetaData.getUploadedFilePath(); - BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(filePath)); + BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath()); try { - InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); + InputStream inputStream = container.readBlob(filePath); IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); From ce63a5a9714dc9724f6512c5f4c80d6f17f262dc Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 20 May 2024 19:39:05 +0530 Subject: [PATCH 066/133] Fix build failure after merge Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 25 ---- .../RemoteClusterStateCleanupManager.java | 22 +-- .../remote/RemoteClusterStateService.java | 125 +++++++++++------- .../main/java/org/opensearch/node/Node.java | 3 +- 4 files changed, 91 insertions(+), 84 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 0760dc1f807f3..56dde258886f5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -795,31 +795,6 @@ public Builder put(String custom, UploadedMetadataAttribute customMetadata) { return this; } - public Builder coordinationMetadata(UploadedMetadataAttribute coordinationMetadata) { - this.coordinationMetadata = coordinationMetadata; - return this; - } - - public Builder settingMetadata(UploadedMetadataAttribute settingsMetadata) { - this.settingsMetadata = settingsMetadata; - return this; - } - - public Builder templatesMetadata(UploadedMetadataAttribute templatesMetadata) { - this.templatesMetadata = templatesMetadata; - return this; - } - - public Builder customMetadataMap(Map customMetadataMap) { - this.customMetadataMap = customMetadataMap; - return this; - } - - public Builder put(String custom, UploadedMetadataAttribute customMetadata) { - this.customMetadataMap.put(custom, customMetadata); - return this; - } - public Builder clusterTerm(long clusterTerm) { this.clusterTerm = clusterTerm; return this; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index b5568cae0572d..8651b5e522233 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -34,12 +34,12 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import static org.opensearch.gateway.remote.RemoteClusterStateService.GLOBAL_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateService.GLOBAL_METADATA_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; /** * A Manager which provides APIs to clean up stale cluster state files and runs an async stale cleanup task @@ -172,7 +172,7 @@ void deleteClusterMetadata( Set staleIndexMetadataPaths = new HashSet<>(); Set staleGlobalMetadataPaths = new HashSet<>(); activeManifestBlobMetadata.forEach(blobMetadata -> { - ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.fetchRemoteClusterMetadataManifest( + ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.getRemoteManifestManager().fetchRemoteClusterMetadataManifest( clusterName, clusterUUID, blobMetadata.name() @@ -190,7 +190,7 @@ void deleteClusterMetadata( logger.info(filesToKeep); }); staleManifestBlobMetadata.forEach(blobMetadata -> { - ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.fetchRemoteClusterMetadataManifest( + ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.getRemoteManifestManager().fetchRemoteClusterMetadataManifest( clusterName, clusterUUID, blobMetadata.name() @@ -252,7 +252,7 @@ void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int mani try { getBlobStoreTransferService().listAllInSortedOrderAsync( ThreadPool.Names.REMOTE_PURGE, - remoteClusterStateService.getManifestFolderPath(clusterName, clusterUUID), + remoteClusterStateService.getRemoteManifestManager().getManifestFolderPath(clusterName, clusterUUID), MANIFEST_FILE_PREFIX, Integer.MAX_VALUE, new ActionListener<>() { @@ -297,7 +297,7 @@ void deleteStaleUUIDsClusterMetadata(String clusterName, List clusterUUI clusterUUIDs.forEach( clusterUUID -> getBlobStoreTransferService().deleteAsync( ThreadPool.Names.REMOTE_PURGE, - remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), + RemoteClusterStateUtils.getCusterMetadataBasePath(remoteClusterStateService.getBlobStoreRepository(), clusterName, clusterUUID), new ActionListener<>() { @Override public void onResponse(Void unused) { @@ -324,7 +324,7 @@ public void onFailure(Exception e) { void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { logger.debug(String.format(Locale.ROOT, "Deleting stale files from remote - %s", stalePaths)); getBlobStoreTransferService().deleteBlobs( - remoteClusterStateService.getCusterMetadataBasePath(clusterName, clusterUUID), + RemoteClusterStateUtils.getCusterMetadataBasePath(remoteClusterStateService.getBlobStoreRepository(), clusterName, clusterUUID), stalePaths ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 324b51adbec11..a42dda35bbc80 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -131,6 +131,7 @@ public class RemoteClusterStateService implements Closeable { private final Settings settings; private final LongSupplier relativeTimeNanosSupplier; private final ThreadPool threadpool; + private final List indexMetadataUploadListeners; private BlobStoreRepository blobStoreRepository; private BlobStoreTransferService blobStoreTransferService; private RemoteRoutingTableService remoteRoutingTableService; @@ -145,7 +146,6 @@ public class RemoteClusterStateService implements Closeable { private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; private TimeValue staleFileCleanupInterval; - private AsyncStaleFileDeletion staleFileDeletionTask; private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " @@ -174,7 +174,8 @@ public RemoteClusterStateService( Settings settings, ClusterService clusterService, LongSupplier relativeTimeNanosSupplier, - ThreadPool threadPool + ThreadPool threadPool, + List indexMetadataUploadListeners ) { assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled"; logger.info("REMOTE STATE ENABLED"); @@ -184,7 +185,7 @@ public RemoteClusterStateService( this.settings = settings; this.relativeTimeNanosSupplier = relativeTimeNanosSupplier; this.threadpool = threadPool; - ClusterSettings clusterSettings = clusterService.getClusterSettings(); + clusterSettings = clusterService.getClusterSettings(); this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); this.remoteStateStats = new RemotePersistenceStats(); @@ -197,7 +198,6 @@ public RemoteClusterStateService( logger.info("REMOTE ROUTING DISABLED"); } this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); - clusterSettings.addSettingsUpdateConsumer(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, this::updateCleanupInterval); this.lastCleanupAttemptState = 0; this.isClusterManagerNode = DiscoveryNode.isClusterManagerNode(settings); this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService); @@ -359,10 +359,6 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousStateIndexMetadataByName.put(indexMetadata.getIndex().getName(), indexMetadata); } - final Map allUploadedIndexMetadata = previousManifest.getIndices() - .stream() - .collect(Collectors.toMap(UploadedIndexMetadata::getIndexName, Function.identity())); - uploadedMetadataResults = writeMetadataInParallel( clusterState, @@ -391,7 +387,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( customsToBeDeletedFromRemote.keySet().forEach(allUploadedCustomMap::remove); indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove); - final ClusterMetadataManifest manifest = uploadManifest( + final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), previousManifest.getPreviousClusterUUID(), @@ -401,8 +397,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUploadForSplitGlobalMetadata || !customsToUpload.isEmpty() ? allUploadedCustomMap : previousManifest.getCustomMetadataMap(), - firstUpload || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), - firstUpload || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), + firstUploadForSplitGlobalMetadata || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), + firstUploadForSplitGlobalMetadata || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), new ClusterStateDiffManifest(clusterState, previousClusterState), routingIndexMetadata, false ); @@ -461,6 +457,31 @@ public ClusterMetadataManifest writeIncrementalMetadata( return manifest; } + private Map getUpdatedCustoms(ClusterState currentState, ClusterState previousState) { + if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) { + return new HashMap<>(); + } + Map updatedCustom = new HashMap<>(); + Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); + for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { + if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { + if (currentCustoms.contains(cursor.getKey()) + && !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { + // If the custom metadata is updated, we need to upload the new version. + updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); + } + currentCustoms.remove(cursor.getKey()); + } + } + for (String custom : currentCustoms) { + Metadata.Custom cursor = currentState.metadata().custom(custom); + if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) { + updatedCustom.put(custom, cursor); + } + } + return updatedCustom; + } + private UploadedMetadataResults writeMetadataInParallel( ClusterState clusterState, List indexToUpload, @@ -665,44 +686,45 @@ private void invokeIndexMetadataUploadListeners( } - - /** - * Allows async Upload of IndexMetadata to remote - * - * @param clusterState current ClusterState - * @param indexMetadata {@link IndexMetadata} to upload - * @param latchedActionListener listener to respond back on after upload finishes - */ - private void writeIndexMetadataAsync( - ClusterState clusterState, - IndexMetadata indexMetadata, - LatchedActionListener latchedActionListener - ) throws IOException { - final BlobContainer indexMetadataContainer = indexMetadataContainer( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID(), - indexMetadata.getIndexUUID() - ); - final String indexMetadataFilename = indexMetadataFileName(indexMetadata); - ActionListener completionListener = ActionListener.wrap( - resp -> latchedActionListener.onResponse( - new UploadedIndexMetadata( - indexMetadata.getIndex().getName(), - indexMetadata.getIndexUUID(), - indexMetadataContainer.path().buildAsString() + indexMetadataFilename - ) + private ActionListener getIndexMetadataUploadActionListener( + List newIndexMetadataList, + Map prevIndexMetadataByName, + CountDownLatch latch, + List exceptionList, + String listenerName + ) { + long startTime = System.nanoTime(); + return new LatchedActionListener<>( + ActionListener.wrap( + ignored -> logger.trace( + new ParameterizedMessage( + "listener={} : Invoked successfully with indexMetadataList={} prevIndexMetadataList={} tookTimeNs={}", + listenerName, + newIndexMetadataList, + prevIndexMetadataByName.values(), + (System.nanoTime() - startTime) + ) + ), + ex -> { + logger.error( + new ParameterizedMessage( + "listener={} : Exception during invocation with indexMetadataList={} prevIndexMetadataList={} tookTimeNs={}", + listenerName, + newIndexMetadataList, + prevIndexMetadataByName.values(), + (System.nanoTime() - startTime) + ), + ex + ); + exceptionList.add(ex); + } ), - ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) + latch ); + } - INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( - indexMetadata, - indexMetadataContainer, - indexMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); + public RemoteManifestManager getRemoteManifestManager() { + return remoteManifestManager; } @Nullable @@ -805,6 +827,10 @@ ThreadPool getThreadpool() { return threadpool; } + BlobStoreRepository getBlobStoreRepository() { + return blobStoreRepository; + } + BlobStore getBlobStore() { return blobStoreRepository.blobStore(); } @@ -952,7 +978,12 @@ public String getLastKnownUUIDFromRemote(String clusterName) { } } - private Set getAllClusterUUIDs(String clusterName) throws IOException { + + public RemoteClusterStateCleanupManager getCleanupManager() { + return remoteClusterStateCleanupManager; + } + + Set getAllClusterUUIDs(String clusterName) throws IOException { Map clusterUUIDMetadata = clusterUUIDContainer(blobStoreRepository, clusterName).children(); if (clusterUUIDMetadata == null) { return Collections.emptySet(); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 44a08b6e559f5..c0e058e6cc984 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -744,7 +744,8 @@ protected Node( settings, clusterService, threadPool::preciseRelativeTimeInNanos, - threadPool + threadPool, + List.of(remoteIndexPathUploader) ); remoteClusterStateCleanupManager = remoteClusterStateService.getCleanupManager(); } else { From eeff23f8d1d64b16e1ae01aa760dc568dab24820 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 20 May 2024 18:18:45 +0530 Subject: [PATCH 067/133] Added support for reading cluster state files in parallel Signed-off-by: Shivansh Arora --- .../org/opensearch/common/io/Streams.java | 7 + .../common/settings/ClusterSettings.java | 2 +- .../RemoteClusterStateAttributesManager.java | 26 +- .../remote/RemoteClusterStateService.java | 322 ++++++++++++------ .../remote/RemoteClusterStateUtils.java | 24 ++ .../remote/RemoteGlobalMetadataManager.java | 27 +- .../remote/RemoteIndexMetadataManager.java | 21 +- .../blobstore/ChecksumBlobStoreFormat.java | 23 +- .../RemoteGlobalMetadataManagerTests.java | 5 +- 9 files changed, 349 insertions(+), 108 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/io/Streams.java b/server/src/main/java/org/opensearch/common/io/Streams.java index f0d8f8b8bedbf..bb70d1901e6a7 100644 --- a/server/src/main/java/org/opensearch/common/io/Streams.java +++ b/server/src/main/java/org/opensearch/common/io/Streams.java @@ -44,10 +44,12 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; +import java.io.SequenceInputStream; import java.io.StringWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Consumer; @@ -221,6 +223,11 @@ public static BytesReference readFully(InputStream in) throws IOException { return out.bytes(); } + public static BytesReference readStreamList(List streams) throws IOException { + SequenceInputStream sis = new SequenceInputStream(Collections.enumeration(streams)); + return readFully(sis); + } + /** * Limits the given input stream to the provided number of bytes */ diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 7afa99cac4f02..20ed4558ae603 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -718,7 +718,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, - RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING, + RemoteClusterStateService.REMOTE_STATE_READ_TIMEOUT_SETTING, RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteGlobalMetadataManager.GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, RemoteManifestManager.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index d18db32c3f61b..6dee7e057e123 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -21,6 +21,7 @@ import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -30,6 +31,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; public class RemoteClusterStateAttributesManager { + public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; public static final String DISCOVERY_NODES = "nodes"; public static final String CLUSTER_BLOCKS = "blocks"; public static final String CUSTOM_PREFIX = "custom"; @@ -45,9 +47,11 @@ public class RemoteClusterStateAttributesManager { ); public static final int CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION = 1; private final BlobStoreRepository blobStoreRepository; + private final ThreadPool threadPool; - RemoteClusterStateAttributesManager(BlobStoreRepository repository) { - blobStoreRepository = repository; + RemoteClusterStateAttributesManager(BlobStoreRepository repository, ThreadPool threadPool) { + this.blobStoreRepository = repository; + this.threadPool = threadPool; } /** @@ -78,6 +82,24 @@ CheckedRunnable getAsyncMetadataWriteAction( ); } + public CheckedRunnable getAsyncMetadataReadAction( + String clusterName, + String clusterUUID, + String component, + String uploadedFilename, + ChecksumBlobStoreFormat componentBlobStore, + LatchedActionListener listener + ) { + String[] splitFilename = uploadedFilename.split("/"); + return () -> componentBlobStore.readAsync( + clusterStateAttributeContainer(clusterName, clusterUUID), + splitFilename[splitFilename.length -1], + blobStoreRepository.getNamedXContentRegistry(), + threadPool.executor(ThreadPool.Names.GENERIC), + ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure) + ); + } + public ToXContent readMetadata(ChecksumBlobStoreFormat componentMetadataBlobStore, String clusterName, String clusterUUID, String fileName) { final BlobContainer remoteStateAttributeContainer = clusterStateAttributeContainer(clusterName, clusterUUID); try { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index a42dda35bbc80..36731b8de8e49 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -13,6 +13,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; @@ -25,8 +26,6 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.blobstore.BlobMetadata; -import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; @@ -36,14 +35,15 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; import java.io.Closeable; @@ -51,7 +51,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -60,7 +59,6 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.LongSupplier; @@ -68,30 +66,27 @@ import java.util.stream.Collectors; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteReadResult; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_DELIMITER; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -115,17 +110,16 @@ public class RemoteClusterStateService implements Closeable { Property.Final ); - /** - * Setting to specify the interval to do run stale file cleanup job - */ - public static final Setting REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING = Setting.timeSetting( - "cluster.remote_store.state.cleanup_interval", - TimeValue.timeValueMinutes(5), - TimeValue.timeValueMillis(-1), - Property.NodeScope, - Property.Dynamic + public static final TimeValue REMOTE_STATE_READ_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); + + public static final Setting REMOTE_STATE_READ_TIMEOUT_SETTING = Setting.timeSetting( + "cluster.remote_store.state.read_timeout", + REMOTE_STATE_READ_TIMEOUT_DEFAULT, + Setting.Property.Dynamic, + Setting.Property.NodeScope ); + private TimeValue remoteStateReadTimeout; private final String nodeId; private final Supplier repositoriesService; private final Settings settings; @@ -133,11 +127,9 @@ public class RemoteClusterStateService implements Closeable { private final ThreadPool threadpool; private final List indexMetadataUploadListeners; private BlobStoreRepository blobStoreRepository; - private BlobStoreTransferService blobStoreTransferService; private RemoteRoutingTableService remoteRoutingTableService; private volatile TimeValue slowWriteLoggingThreshold; - private final AtomicBoolean deleteStaleMetadataRunning = new AtomicBoolean(false); private final RemotePersistenceStats remoteStateStats; private RemoteClusterStateCleanupManager remoteClusterStateCleanupManager; private RemoteIndexMetadataManager remoteIndexMetadataManager; @@ -145,14 +137,11 @@ public class RemoteClusterStateService implements Closeable { private RemoteClusterStateAttributesManager remoteClusterStateAttributesManager; private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; - private TimeValue staleFileCleanupInterval; private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; - public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. @@ -188,6 +177,8 @@ public RemoteClusterStateService( clusterSettings = clusterService.getClusterSettings(); this.slowWriteLoggingThreshold = clusterSettings.get(SLOW_WRITE_LOGGING_THRESHOLD); clusterSettings.addSettingsUpdateConsumer(SLOW_WRITE_LOGGING_THRESHOLD, this::setSlowWriteLoggingThreshold); + this.remoteStateReadTimeout = clusterSettings.get(REMOTE_STATE_READ_TIMEOUT_SETTING); + clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteClusterStateEnabled); this.remoteStateStats = new RemotePersistenceStats(); if(isRemoteRoutingTableEnabled(settings)) { @@ -197,7 +188,6 @@ public RemoteClusterStateService( } else { logger.info("REMOTE ROUTING DISABLED"); } - this.staleFileCleanupInterval = clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING); this.lastCleanupAttemptState = 0; this.isClusterManagerNode = DiscoveryNode.isClusterManagerNode(settings); this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService); @@ -297,7 +287,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( assert previousClusterState.metadata().coordinationMetadata().term() == clusterState.metadata().coordinationMetadata().term(); final Map customsToBeDeletedFromRemote = new HashMap<>(previousManifest.getCustomMetadataMap()); - final Map customsToUpload = getUpdatedCustoms(clusterState, previousClusterState); + final Map customsToUpload = remoteGlobalMetadataManager.getUpdatedCustoms(clusterState, previousClusterState); final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); for (final String custom : clusterState.metadata().customs().keySet()) { // remove all the customs which are present currently @@ -457,31 +447,6 @@ public ClusterMetadataManifest writeIncrementalMetadata( return manifest; } - private Map getUpdatedCustoms(ClusterState currentState, ClusterState previousState) { - if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) { - return new HashMap<>(); - } - Map updatedCustom = new HashMap<>(); - Set currentCustoms = new HashSet<>(currentState.metadata().customs().keySet()); - for (Map.Entry cursor : previousState.metadata().customs().entrySet()) { - if (cursor.getValue().context().contains(Metadata.XContentContext.GATEWAY)) { - if (currentCustoms.contains(cursor.getKey()) - && !cursor.getValue().equals(currentState.metadata().custom(cursor.getKey()))) { - // If the custom metadata is updated, we need to upload the new version. - updatedCustom.put(cursor.getKey(), currentState.metadata().custom(cursor.getKey())); - } - currentCustoms.remove(cursor.getKey()); - } - } - for (String custom : currentCustoms) { - Metadata.Custom cursor = currentState.metadata().custom(custom); - if (cursor.context().contains(Metadata.XContentContext.GATEWAY)) { - updatedCustom.put(custom, cursor); - } - } - return updatedCustom; - } - private UploadedMetadataResults writeMetadataInParallel( ClusterState clusterState, List indexToUpload, @@ -794,26 +759,13 @@ public void start() { if(this.remoteRoutingTableService != null) { this.remoteRoutingTableService.start(); } - remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings); - remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(blobStoreRepository); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings, threadpool); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, threadpool); + remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(blobStoreRepository, threadpool); remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); remoteClusterStateCleanupManager.start(); } - private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) { - final Optional latestManifest = remoteManifestManager.getLatestClusterMetadataManifest( - clusterName, - clusterUUID - ); - if (!latestManifest.isPresent()) { - final String previousClusterUUID = getLastKnownUUIDFromRemote(clusterName); - assert !clusterUUID.equals(previousClusterUUID) : "Last cluster UUID is same current cluster UUID"; - return previousClusterUUID; - } - return latestManifest.get().getPreviousClusterUUID(); - } - private void setSlowWriteLoggingThreshold(TimeValue slowWriteLoggingThreshold) { this.slowWriteLoggingThreshold = slowWriteLoggingThreshold; } @@ -842,7 +794,7 @@ BlobStore getBlobStore() { * @param clusterName name of the cluster * @return {@link IndexMetadata} */ - public ClusterState getLatestClusterState(String clusterName, String clusterUUID) { + public ClusterState getLatestClusterState(String clusterName, String clusterUUID) throws IOException { start(); Optional clusterMetadataManifest = remoteManifestManager.getLatestClusterMetadataManifest( clusterName, @@ -857,42 +809,218 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID return getClusterStateForManifest(clusterName, clusterMetadataManifest.get(), nodeId); } - public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId) { - // todo make this async - // Fetch Global Metadata - Metadata globalMetadata = remoteGlobalMetadataManager.getGlobalMetadata(clusterName, manifest.getClusterUUID(), manifest); + private ClusterState readClusterStateInParallel( + ClusterState previousState, + ClusterMetadataManifest manifest, + String clusterName, + String clusterUUID, + String localNodeId, + List indicesToRead, + Map customToRead, + boolean readCoordinationMetadata, + boolean readSettingsMetadata, + boolean readTemplatesMetadata, + boolean readDiscoveryNodes, + boolean readClusterBlocks + ) throws IOException { + int totalReadTasks = indicesToRead.size() + customToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + (readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0); + CountDownLatch latch = new CountDownLatch(totalReadTasks); + List> asyncMetadataReadActions = new ArrayList<>(); + List readResults = new ArrayList<>(); + List exceptionList = Collections.synchronizedList(new ArrayList<>(totalReadTasks)); - // Fetch Index Metadata - Map indices = remoteIndexMetadataManager.getIndexMetadataMap( - clusterName, - manifest.getClusterUUID(), - manifest + LatchedActionListener listener = new LatchedActionListener<>( + ActionListener.wrap( + response -> { + logger.info("Successfully read cluster state component from remote"); + readResults.add(response); + }, + ex -> { + logger.error("Failed to read cluster state from remote", ex); + exceptionList.add(ex); + } + ), + latch ); - DiscoveryNodes discoveryNodes = (DiscoveryNodes) remoteClusterStateAttributesManager.readMetadata(DISCOVERY_NODES_FORMAT, clusterName, manifest.getClusterUUID(), manifest.getDiscoveryNodesMetadata().getUploadedFilename()); - Map indexMetadataMap = new HashMap<>(); - indices.values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); }); - - if(remoteRoutingTableService != null) { - logger.info(" The Remote Routing Table is fetching the Full Routing Table"); - RoutingTable routingTable = remoteRoutingTableService.getFullRoutingTable(manifest.getRoutingTableVersion(), manifest.getIndicesRouting()); - return ClusterState.builder(ClusterState.EMPTY_STATE) - .version(manifest.getStateVersion()) - .stateUUID(manifest.getStateUUID()) - .nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNodeId)) - .metadata(Metadata.builder(globalMetadata).indices(indexMetadataMap).build()) - .routingTable(routingTable) - .build(); + for (UploadedIndexMetadata indexMetadata : indicesToRead) { + asyncMetadataReadActions.add( + remoteIndexMetadataManager.getAsyncIndexMetadataReadAction( + clusterName, + clusterUUID, + indexMetadata.getUploadedFilename(), + listener + ) + ); } - return ClusterState.builder(ClusterState.EMPTY_STATE) + for (Map.Entry entry : customToRead.entrySet()) { + asyncMetadataReadActions.add( + remoteGlobalMetadataManager.getAsyncMetadataReadAction( + clusterName, + clusterUUID, + CUSTOM_METADATA, + entry.getKey(), + entry.getValue().getUploadedFilename(), + new ChecksumBlobStoreFormat("custom", METADATA_NAME_FORMAT, (parser -> Metadata.Custom.fromXContent((XContentParser) parser, entry.getKey()))), + listener + ) + ); + } + + if (readCoordinationMetadata) { + asyncMetadataReadActions.add( + remoteGlobalMetadataManager.getAsyncMetadataReadAction( + clusterName, + clusterUUID, + COORDINATION_METADATA, + COORDINATION_METADATA, + manifest.getCoordinationMetadata().getUploadedFilename(), + COORDINATION_METADATA_FORMAT, + listener + ) + ); + } + + if (readSettingsMetadata) { + asyncMetadataReadActions.add( + remoteGlobalMetadataManager.getAsyncMetadataReadAction( + clusterName, + clusterUUID, + SETTING_METADATA, + SETTING_METADATA, + manifest.getSettingsMetadata().getUploadedFilename(), + SETTINGS_METADATA_FORMAT, + listener + ) + ); + } + + if (readTemplatesMetadata) { + asyncMetadataReadActions.add( + remoteGlobalMetadataManager.getAsyncMetadataReadAction( + clusterName, + clusterUUID, + TEMPLATES_METADATA, + TEMPLATES_METADATA, + manifest.getTemplatesMetadata().getUploadedFilename(), + TEMPLATES_METADATA_FORMAT, + listener + ) + ); + } + + if (readDiscoveryNodes) { + asyncMetadataReadActions.add( + remoteClusterStateAttributesManager.getAsyncMetadataReadAction( + clusterName, + clusterUUID, + DISCOVERY_NODES, + manifest.getDiscoveryNodesMetadata().getUploadedFilename(), + DISCOVERY_NODES_FORMAT, + listener + ) + ); + } + + if (readClusterBlocks) { + asyncMetadataReadActions.add( + remoteClusterStateAttributesManager.getAsyncMetadataReadAction( + clusterName, + clusterUUID, + CLUSTER_BLOCKS, + manifest.getClusterBlocksMetadata().getUploadedFilename(), + CLUSTER_BLOCKS_FORMAT, + listener + ) + ); + } + + for (CheckedRunnable asyncMetadataReadAction : asyncMetadataReadActions) { + asyncMetadataReadAction.run(); + } + + try { + if (latch.await(this.remoteStateReadTimeout.getMillis(), TimeUnit.MILLISECONDS) == false) { + RemoteStateTransferException exception = new RemoteStateTransferException( + "Timed out waiting to read cluster state from remote within timeout " + this.remoteStateReadTimeout + ); + exceptionList.forEach(exception::addSuppressed); + throw exception; + } + } catch (InterruptedException e) { + exceptionList.forEach(e::addSuppressed); + RemoteStateTransferException ex = new RemoteStateTransferException("Interrupted while waiting to read cluster state from metadata"); + Thread.currentThread().interrupt(); + throw ex; + } + + if (!exceptionList.isEmpty()) { + RemoteStateTransferException exception = new RemoteStateTransferException("Exception during reading cluster state from remote"); + exceptionList.forEach(exception::addSuppressed); + throw exception; + } + ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); + AtomicReference discoveryNodesBuilder = new AtomicReference<>(DiscoveryNodes.builder()); + Metadata.Builder metadataBuilder = Metadata.builder(); + metadataBuilder.clusterUUID(manifest.getClusterUUID()); + metadataBuilder.clusterUUIDCommitted(manifest.isClusterUUIDCommitted()); + Map indexMetadataMap = new HashMap<>(); + readResults.forEach(remoteReadResult -> { + switch (remoteReadResult.getComponent()) { + case INDEX_PATH_TOKEN: + IndexMetadata indexMetadata = (IndexMetadata) remoteReadResult.getObj(); + indexMetadataMap.put(indexMetadata.getIndex().getName(), indexMetadata); + break; + case CUSTOM_METADATA: + metadataBuilder.putCustom(remoteReadResult.getComponentName(), (Metadata.Custom) remoteReadResult.getObj()); + break; + case COORDINATION_METADATA: + metadataBuilder.coordinationMetadata((CoordinationMetadata) remoteReadResult.getObj()); + break; + case SETTING_METADATA: + metadataBuilder.persistentSettings((Settings) remoteReadResult.getObj()); + break; + case TEMPLATES_METADATA: + metadataBuilder.templates((TemplatesMetadata) remoteReadResult.getObj()); + break; + case CLUSTER_STATE_ATTRIBUTE: + if (remoteReadResult.getComponentName().equals(DISCOVERY_NODES)) { + discoveryNodesBuilder.set(DiscoveryNodes.builder((DiscoveryNodes) remoteReadResult.getObj())); + } else if (remoteReadResult.getComponentName().equals(CLUSTER_BLOCKS)) { + clusterStateBuilder.blocks((ClusterBlocks) remoteReadResult.getObj()); + } + break; + default: + throw new IllegalStateException("Unknown component: " + remoteReadResult.getComponent()); + } + }); + metadataBuilder.indices(indexMetadataMap); + return clusterStateBuilder.metadata(metadataBuilder) + .nodes(discoveryNodesBuilder.get().localNodeId(localNodeId)) .version(manifest.getStateVersion()) .stateUUID(manifest.getStateUUID()) - .nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNodeId)) - .metadata(Metadata.builder(globalMetadata).indices(indexMetadataMap).build()) .build(); } + public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId) throws IOException { + // todo make this async + return readClusterStateInParallel( + ClusterState.EMPTY_STATE, + manifest, + clusterName, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + manifest.getCustomMetadataMap(), + manifest.getCoordinationMetadata() != null, + manifest.getSettingsMetadata() != null, + manifest.getTemplatesMetadata() != null, + manifest.getDiscoveryNodesMetadata() != null, + manifest.getClusterBlocksMetadata() != null + ); + } + public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) { assert manifest.getDiffManifest() != null; ClusterStateDiffManifest diff = manifest.getDiffManifest(); @@ -978,6 +1106,10 @@ public String getLastKnownUUIDFromRemote(String clusterName) { } } + public void setRemoteClusterStateEnabled(TimeValue remoteStateReadTimeout) { + this.remoteStateReadTimeout = remoteStateReadTimeout; + } + public RemoteClusterStateCleanupManager getCleanupManager() { return remoteClusterStateCleanupManager; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 3263cabc50687..3b71afd6026c3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -104,4 +104,28 @@ public UploadedMetadataResults() { this.uploadedClusterBlocks = null; } } + + static class RemoteReadResult { + ToXContent obj; + String component; + String componentName; + + public RemoteReadResult(ToXContent obj, String component, String componentName) { + this.obj = obj; + this.component = component; + this.componentName = componentName; + } + + public ToXContent getObj() { + return obj; + } + + public String getComponent() { + return component; + } + + public String getComponentName() { + return componentName; + } + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 4f650e3bf2954..76346376255b0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -24,13 +24,17 @@ import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; import static java.util.Objects.requireNonNull; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -89,12 +93,14 @@ public class RemoteGlobalMetadataManager { public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; private final BlobStoreRepository blobStoreRepository; + private final ThreadPool threadPool; private volatile TimeValue globalMetadataUploadTimeout; - RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings) { + RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool) { this.blobStoreRepository = blobStoreRepository; this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); + this.threadPool = threadPool; clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); } @@ -129,6 +135,25 @@ CheckedRunnable getAsyncMetadataWriteAction( ); } + CheckedRunnable getAsyncMetadataReadAction( + String clusterName, + String clusterUUID, + String component, + String componentName, + String uploadFilename, + ChecksumBlobStoreFormat componentBlobStore, + LatchedActionListener listener + ) { + String[] splitName = uploadFilename.split("/"); + return () -> componentBlobStore.readAsync( + globalMetadataContainer(clusterName, clusterUUID), + splitName[splitName.length - 1], + blobStoreRepository.getNamedXContentRegistry(), + threadPool.executor(ThreadPool.Names.GENERIC), + ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure) + ); + } + Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { String globalMetadataFileName = clusterMetadataManifest.getGlobalMetadataFileName(); try { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index c034d597817ea..3d50b9d8453fa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -20,6 +20,7 @@ import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; import java.util.HashMap; @@ -53,12 +54,14 @@ public class RemoteIndexMetadataManager { public static final String INDEX_PATH_TOKEN = "index"; private final BlobStoreRepository blobStoreRepository; + private final ThreadPool threadPool; private volatile TimeValue indexMetadataUploadTimeout; - public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings) { + public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool) { this.blobStoreRepository = blobStoreRepository; this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); + this.threadPool = threadPool; clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); } @@ -101,6 +104,22 @@ CheckedRunnable getIndexMetadataAsyncAction( ); } + CheckedRunnable getAsyncIndexMetadataReadAction( + String clusterName, + String clusterUUID, + String uploadedFilename, + LatchedActionListener latchedActionListener + ) { + String[] splitPath = uploadedFilename.split("/"); + return () -> INDEX_METADATA_FORMAT.readAsync( + indexMetadataContainer(clusterName, clusterUUID, splitPath[0]), + splitPath[splitPath.length - 1], + blobStoreRepository.getNamedXContentRegistry(), + threadPool.executor(ThreadPool.Names.GENERIC), + ActionListener.wrap(response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, splitPath[0])), latchedActionListener::onFailure) + ); + } + /** * Fetch index metadata from remote cluster state * diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java index b5812f760ede2..78b8fbeb45852 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -44,9 +44,11 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; +import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -61,12 +63,20 @@ import org.opensearch.gateway.CorruptStateException; import org.opensearch.index.store.exception.ChecksumCombinationException; import org.opensearch.snapshots.SnapshotInfo; +import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; @@ -115,17 +125,16 @@ public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunct * @param name name to be translated into * @return parsed blob object */ - public T read(BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry) throws IOException { - String blobName = blobName(name); + public T read(BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry) throws IOException {String blobName = blobName(name); return deserialize(blobName, namedXContentRegistry, Streams.readFully(blobContainer.readBlob(blobName))); } - public CompletableFuture readAsync(BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry) throws IOException { - return CompletableFuture.supplyAsync(() -> { + public void readAsync(BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry, ExecutorService executorService, ActionListener listener) throws IOException { + executorService.execute(() -> { try { - return ChecksumBlobStoreFormat.this.read(blobContainer, name, namedXContentRegistry); - } catch (IOException e) { - throw new RuntimeException(e); + listener.onResponse(read(blobContainer, name, namedXContentRegistry)); + } catch (Exception e) { + listener.onFailure(e); } }); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java index c9fb7d13e678a..00c8cf3c0df5d 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java @@ -14,6 +14,8 @@ import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; import static org.mockito.Mockito.mock; @@ -21,12 +23,13 @@ public class RemoteGlobalMetadataManagerTests extends OpenSearchTestCase { private RemoteGlobalMetadataManager remoteGlobalMetadataManager; private ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @Before public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); - remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings, threadPool); } @After From 7242dad89cc21c11ff562242eea261007e95f6af Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 21 May 2024 17:01:56 +0530 Subject: [PATCH 068/133] Read diff files asynchronously Signed-off-by: Shivansh Arora --- .../remote/ClusterStateDiffManifest.java | 41 +++++++++---- .../remote/RemoteClusterStateService.java | 58 ++++++++++--------- 2 files changed, 61 insertions(+), 38 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index c184166c490c3..8a09252c99570 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -46,6 +46,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { private final boolean settingsMetadataUpdated; private final boolean templatesMetadataUpdated; private final Map customMetadataUpdated; + private final List customMetadataDeleted; private final List indicesUpdated; private final List indicesDeleted; private final boolean clusterBlocksUpdated; @@ -70,28 +71,35 @@ public class ClusterStateDiffManifest implements ToXContentObject { state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) ); } + customMetadataDeleted = new ArrayList<>(); + for (String custom : previousState.metadata().customs().keySet()) { + if (state.metadata().customs().get(custom) == null) { + customMetadataDeleted.add(custom); + } + } indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); } public ClusterStateDiffManifest(String fromStateUUID, - String toStateUUID, - boolean coordinationMetadataUpdated, - boolean settingsMetadataUpdated, - boolean templatesMetadataUpdated, - Map customMetadataUpdated, - List indicesUpdated, - List indicesDeleted, - boolean clusterBlocksUpdated, - boolean discoveryNodesUpdated, - ListindicesRoutingUpdated, - ListindicesRoutingDeleted) { + String toStateUUID, + boolean coordinationMetadataUpdated, + boolean settingsMetadataUpdated, + boolean templatesMetadataUpdated, + Map customMetadataUpdated, List customMetadataDeleted, + List indicesUpdated, + List indicesDeleted, + boolean clusterBlocksUpdated, + boolean discoveryNodesUpdated, + ListindicesRoutingUpdated, + ListindicesRoutingDeleted) { this.fromStateUUID = fromStateUUID; this.toStateUUID = toStateUUID; this.coordinationMetadataUpdated = coordinationMetadataUpdated; this.settingsMetadataUpdated = settingsMetadataUpdated; this.templatesMetadataUpdated = templatesMetadataUpdated; this.customMetadataUpdated = customMetadataUpdated; + this.customMetadataDeleted = customMetadataDeleted; this.indicesUpdated = indicesUpdated; this.indicesDeleted = indicesDeleted; this.clusterBlocksUpdated = clusterBlocksUpdated; @@ -320,6 +328,10 @@ public Map getCustomMetadataUpdated() { return customMetadataUpdated; } + public List getCustomMetadataDeleted() { + return customMetadataDeleted; + } + public List getIndicesUpdated() { return indicesUpdated; } @@ -355,6 +367,7 @@ public static class Builder { private boolean settingsMetadataUpdated; private boolean templatesMetadataUpdated; private Map customMetadataUpdated; + private List customMetadataDeleted; private List indicesUpdated; private List indicesDeleted; private boolean clusterBlocksUpdated; @@ -393,6 +406,11 @@ public Builder customMetadataUpdated(Map customMetadataUpdated) return this; } + public Builder customMetadataDeleted(List customMetadataDeleted) { + this.customMetadataDeleted = customMetadataDeleted; + return this; + } + public Builder indicesUpdated(List indicesUpdated) { this.indicesUpdated = indicesUpdated; return this; @@ -431,6 +449,7 @@ public ClusterStateDiffManifest build() { settingsMetadataUpdated, templatesMetadataUpdated, customMetadataUpdated, + customMetadataDeleted, indicesUpdated, indicesDeleted, clusterBlocksUpdated, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 36731b8de8e49..58009859479fe 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -962,7 +962,7 @@ private ClusterState readClusterStateInParallel( } ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); AtomicReference discoveryNodesBuilder = new AtomicReference<>(DiscoveryNodes.builder()); - Metadata.Builder metadataBuilder = Metadata.builder(); + Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); metadataBuilder.clusterUUID(manifest.getClusterUUID()); metadataBuilder.clusterUUIDCommitted(manifest.isClusterUUIDCommitted()); Map indexMetadataMap = new HashMap<>(); @@ -996,8 +996,10 @@ private ClusterState readClusterStateInParallel( } }); metadataBuilder.indices(indexMetadataMap); + if (readDiscoveryNodes) { + clusterStateBuilder.nodes(discoveryNodesBuilder.get().localNodeId(localNodeId)) + } return clusterStateBuilder.metadata(metadataBuilder) - .nodes(discoveryNodesBuilder.get().localNodeId(localNodeId)) .version(manifest.getStateVersion()) .stateUUID(manifest.getStateUUID()) .build(); @@ -1021,38 +1023,37 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada ); } - public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) { + public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) throws IOException { assert manifest.getDiffManifest() != null; ClusterStateDiffManifest diff = manifest.getDiffManifest(); - ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); - Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); - - for (String index:diff.getIndicesUpdated()) { - //todo optimize below iteration - Optional uploadedIndexMetadataOptional = manifest.getIndices().stream().filter(idx -> idx.getIndexName().equals(index)).findFirst(); + List updatedIndices = diff.getIndicesUpdated().stream().map(idx -> { + Optional uploadedIndexMetadataOptional = manifest.getIndices().stream().filter(idx2 -> idx2.getIndexName().equals(idx)).findFirst(); assert uploadedIndexMetadataOptional.isPresent() == true; - IndexMetadata indexMetadata = remoteIndexMetadataManager.getIndexMetadata(clusterName, manifest.getClusterUUID(), uploadedIndexMetadataOptional.get()); - metadataBuilder.put(indexMetadata, false); + return uploadedIndexMetadataOptional.get(); + }).collect(Collectors.toList()); + Map updatedCustomMetadata = new HashMap<>(); + for (String customType : diff.getCustomMetadataUpdated().keySet()) { + updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); } + ClusterState updatedClusterState = readClusterStateInParallel( + previousState, + manifest, + clusterName, + manifest.getClusterUUID(), + localNodeId, + updatedIndices, + updatedCustomMetadata, + diff.isCoordinationMetadataUpdated(), + diff.isSettingsMetadataUpdated(), + diff.isTemplatesMetadataUpdated(), + diff.isDiscoveryNodesUpdated(), + diff.isClusterBlocksUpdated() + ); + ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); + Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); for (String index:diff.getIndicesDeleted()) { metadataBuilder.remove(index); } - if (diff.isCoordinationMetadataUpdated()) { - CoordinationMetadata coordinationMetadata = remoteGlobalMetadataManager.getCoordinationMetadata(clusterName, manifest.getClusterUUID(), manifest.getCoordinationMetadata().getUploadedFilename()); - metadataBuilder.coordinationMetadata(coordinationMetadata); - } - if (diff.isSettingsMetadataUpdated()) { - Settings settings = remoteGlobalMetadataManager.getSettingsMetadata(clusterName, manifest.getClusterUUID(), manifest.getSettingsMetadata().getUploadedFilename()); - metadataBuilder.persistentSettings(settings); - } - if (diff.isTemplatesMetadataUpdated()) { - TemplatesMetadata templatesMetadata = remoteGlobalMetadataManager.getTemplatesMetadata(clusterName, manifest.getClusterUUID(), manifest.getTemplatesMetadata().getUploadedFilename()); - metadataBuilder.templates(templatesMetadata); - } - if (diff.isDiscoveryNodesUpdated()) { - DiscoveryNodes discoveryNodes = (DiscoveryNodes) remoteClusterStateAttributesManager.readMetadata(DISCOVERY_NODES_FORMAT, clusterName, manifest.getClusterUUID(), manifest.getDiscoveryNodesMetadata().getUploadedFilename()); - clusterStateBuilder.nodes(DiscoveryNodes.builder(discoveryNodes).localNodeId(localNodeId)); - } if (diff.getCustomMetadataUpdated() != null) { for (String customType : diff.getCustomMetadataUpdated().keySet()) { Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(clusterName, manifest.getClusterUUID(), @@ -1060,6 +1061,9 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata metadataBuilder.putCustom(customType, custom); } } + for (String customType : diff.getCustomMetadataDeleted()) { + metadataBuilder.removeCustom(customType); + } // Constructing Routing Table if(remoteRoutingTableService!= null){ From 44e1d251bec12ed12558ff2015bae3f51757bb5c Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 21 May 2024 17:04:45 +0530 Subject: [PATCH 069/133] add missed semicolon Signed-off-by: Shivansh Arora --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 58009859479fe..d21b38dc389fd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -997,7 +997,7 @@ private ClusterState readClusterStateInParallel( }); metadataBuilder.indices(indexMetadataMap); if (readDiscoveryNodes) { - clusterStateBuilder.nodes(discoveryNodesBuilder.get().localNodeId(localNodeId)) + clusterStateBuilder.nodes(discoveryNodesBuilder.get().localNodeId(localNodeId)); } return clusterStateBuilder.metadata(metadataBuilder) .version(manifest.getStateVersion()) From 9982d793f03155c160a4a27689c62dfba066319c Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 21 May 2024 17:42:16 +0530 Subject: [PATCH 070/133] Fix tests after merge Signed-off-by: Shivansh Arora --- .../RemoteClusterStateCleanupManager.java | 1 - .../remote/ClusterMetadataManifestTests.java | 122 +++++++----------- ...RemoteClusterStateCleanupManagerTests.java | 25 ++-- .../RemoteClusterStateServiceTests.java | 112 ++-------------- .../RemoteIndexMetadataManagerTests.java | 3 +- .../remote/RemoteIndexPathUploaderTests.java | 5 +- 6 files changed, 71 insertions(+), 197 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 8651b5e522233..0d6c9b715a331 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -187,7 +187,6 @@ void deleteClusterMetadata( filesToKeep.add(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); clusterMetadataManifest.getCustomMetadataMap().values().forEach(attribute -> filesToKeep.add(attribute.getUploadedFilename())); } - logger.info(filesToKeep); }); staleManifestBlobMetadata.forEach(blobMetadata -> { ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.getRemoteManifestManager().fetchRemoteClusterMetadataManifest( diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index df9ccc5bf211e..479d92dd9cdfb 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -33,6 +33,7 @@ import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; public class ClusterMetadataManifestTests extends OpenSearchTestCase { @@ -89,9 +90,9 @@ public void testClusterMetadataManifestXContentV1() throws IOException { } } - public void testClusterMetadataManifestXContentV1() throws IOException { + public void testClusterMetadataManifestXContent() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder() .clusterTerm(1L) .stateVersion(1L) .clusterUUID("test-cluster-uuid") @@ -99,42 +100,14 @@ public void testClusterMetadataManifestXContentV1() throws IOException { .opensearchVersion(Version.CURRENT) .nodeId("test-node-id") .committed(false) - .codecVersion(CODEC_V1) - .globalMetadataFileName("test-global-metadata-file") + .codecVersion(CODEC_V2) .indices(Collections.singletonList(uploadedIndexMetadata)) .previousClusterUUID("prev-cluster-uuid") .clusterUUIDCommitted(true) - .build(); - final XContentBuilder builder = JsonXContent.contentBuilder(); - builder.startObject(); - originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV1(parser); - assertEquals(originalManifest, fromXContentManifest); - } - } - - public void testClusterMetadataManifestXContent() throws IOException { - UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V2, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - new UploadedMetadataAttribute(RemoteGlobalMetadataManager.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteGlobalMetadataManager.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteGlobalMetadataManager.TEMPLATES_METADATA, "templates-file"), - Collections.unmodifiableList( + .coordinationMetadata(new UploadedMetadataAttribute(RemoteGlobalMetadataManager.COORDINATION_METADATA, "coordination-file")) + .settingMetadata(new UploadedMetadataAttribute(RemoteGlobalMetadataManager.SETTING_METADATA, "setting-file")) + .templatesMetadata(new UploadedMetadataAttribute(RemoteGlobalMetadataManager.TEMPLATES_METADATA, "templates-file")) + .customMetadataMap(Collections.unmodifiableList( Arrays.asList( new UploadedMetadataAttribute( RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER @@ -151,11 +124,8 @@ public void testClusterMetadataManifestXContent() throws IOException { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - null, - null, - null - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()))); + ClusterMetadataManifest originalManifest = manifestBuilder.build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -168,23 +138,22 @@ public void testClusterMetadataManifestXContent() throws IOException { } public void testClusterMetadataManifestSerializationEqualsHashCode() { - ClusterMetadataManifest initialManifest = new ClusterMetadataManifest( - 1337L, - 7L, - "HrYF3kP5SmSPWtKlWhnNSA", - "6By9p9G0Rv2MmFYJcPAOgA", - Version.CURRENT, - "B10RX1f5RJenMQvYccCgSQ", - true, - 2, - null, - randomUploadedIndexMetadataList(), - "yfObdx8KSMKKrXf8UyHhM", - true, - new UploadedMetadataAttribute(RemoteGlobalMetadataManager.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteGlobalMetadataManager.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteGlobalMetadataManager.TEMPLATES_METADATA, "templates-file"), - Collections.unmodifiableList( + ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder(); + manifestBuilder.clusterTerm(1337L) + .stateVersion(7L) + .clusterUUID("HrYF3kP5SmSPWtKlWhnNSA") + .stateUUID("6By9p9G0Rv2MmFYJcPAOgA") + .opensearchVersion(Version.CURRENT) + .nodeId("B10RX1f5RJenMQvYccCgSQ") + .committed(true) + .codecVersion(2) + .indices(randomUploadedIndexMetadataList()) + .previousClusterUUID("yfObdx8KSMKKrXf8UyHhM") + .clusterUUIDCommitted(true) + .coordinationMetadata(new UploadedMetadataAttribute(RemoteGlobalMetadataManager.COORDINATION_METADATA, "coordination-file")) + .settingMetadata(new UploadedMetadataAttribute(RemoteGlobalMetadataManager.SETTING_METADATA, "setting-file")) + .templatesMetadata(new UploadedMetadataAttribute(RemoteGlobalMetadataManager.TEMPLATES_METADATA, "templates-file")) + .customMetadataMap(Collections.unmodifiableList( Arrays.asList( new UploadedMetadataAttribute( RemoteGlobalMetadataManager.CUSTOM_METADATA + RemoteGlobalMetadataManager.CUSTOM_DELIMITER @@ -201,11 +170,8 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { "custom--weighted_routing_netadata-file" ) ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - null, - null, - null - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()))); + ClusterMetadataManifest initialManifest = manifestBuilder.build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialManifest, @@ -346,22 +312,22 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { public void testClusterMetadataManifestXContentV2() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V2, - "test-metadata", - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - 1L, - Collections.singletonList(uploadedIndexMetadata) - ); + ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder(); + manifestBuilder.clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(CODEC_V2) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .routingTableVersion(1L) + .indicesRouting(Collections.singletonList(uploadedIndexMetadata)); + + ClusterMetadataManifest originalManifest = manifestBuilder.build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 1a2d5c2ab6751..6d1a8e0207851 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -59,18 +59,19 @@ import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateCleanupManager.SKIP_CLEANUP_STATE_CHANGES; -import static org.opensearch.gateway.remote.RemoteClusterStateService.CLUSTER_STATE_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateService.GLOBAL_METADATA_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterStateService.encodeString; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.encodeString; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -207,7 +208,7 @@ public void testDeleteClusterMetadata() throws IOException { .templatesMetadata(templateMetadataUpdated) .build(); - when(remoteClusterStateService.fetchRemoteClusterMetadataManifest(eq(clusterName), eq(clusterUUID), any())) + when(remoteClusterStateService.getRemoteManifestManager().fetchRemoteClusterMetadataManifest(eq(clusterName), eq(clusterUUID), any())) .thenReturn(manifest4, manifest5, manifest1, manifest2, manifest3); BlobContainer container = mock(BlobContainer.class); when(blobStore.blobContainer(any())).thenReturn(container); @@ -273,7 +274,7 @@ public void testDeleteStaleClusterUUIDs() throws IOException { ).thenReturn(List.of(new PlainBlobMetadata("mainfest3", 1L))); Set uuids = new HashSet<>(Arrays.asList("cluster-uuid1", "cluster-uuid2", "cluster-uuid3")); when(remoteClusterStateService.getAllClusterUUIDs(any())).thenReturn(uuids); - when(remoteClusterStateService.getCusterMetadataBasePath(any(), any())).then( + when(getCusterMetadataBasePath(any(), any(), any())).then( invocationOnMock -> blobPath.add(encodeString(invocationOnMock.getArgument(0))) .add(CLUSTER_STATE_PATH_TOKEN) .add((String) invocationOnMock.getArgument(1)) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index e5bb69f7e15c5..acd8e19102b00 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -89,7 +89,6 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; @@ -98,7 +97,6 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; @@ -125,6 +123,7 @@ public class RemoteClusterStateServiceTests extends OpenSearchTestCase { private RepositoriesService repositoriesService; private BlobStoreRepository blobStoreRepository; private BlobStore blobStore; + private Settings settings; private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @Before @@ -144,7 +143,7 @@ public void setup() { "remote_store_repository" ); - Settings settings = Settings.builder() + settings = Settings.builder() .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") .put(stateRepoTypeAttributeKey, FsRepository.TYPE) .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") @@ -173,7 +172,8 @@ public void setup() { settings, clusterService, () -> 0L, - threadPool + threadPool, + List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) ); } @@ -201,7 +201,8 @@ public void testFailInitializationWhenRemoteStateDisabled() { settings, clusterService, () -> 0L, - threadPool + threadPool, + List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) ) ); } @@ -1278,102 +1279,6 @@ public void testRemoteStateStats() throws IOException { assertEquals(0, remoteClusterStateService.getStats().getFailedCount()); } - public void testRemoteStateCleanupFailureStats() throws IOException { - BlobContainer blobContainer = mock(BlobContainer.class); - doThrow(IOException.class).when(blobContainer).delete(); - when(blobStore.blobContainer(any())).thenReturn(blobContainer); - BlobPath blobPath = new BlobPath().add("random-path"); - when((blobStoreRepository.basePath())).thenReturn(blobPath); - remoteClusterStateService.start(); - remoteClusterStateService.deleteStaleUUIDsClusterMetadata("cluster1", Arrays.asList("cluster-uuid1")); - try { - assertBusy(() -> { - // wait for stats to get updated - assertTrue(remoteClusterStateService.getStats() != null); - assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); - assertEquals(1, remoteClusterStateService.getStats().getCleanupAttemptFailedCount()); - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public void testSingleConcurrentExecutionOfStaleManifestCleanup() throws Exception { - BlobContainer blobContainer = mock(BlobContainer.class); - BlobPath blobPath = new BlobPath().add("random-path"); - when((blobStoreRepository.basePath())).thenReturn(blobPath); - when(blobStore.blobContainer(any())).thenReturn(blobContainer); - - CountDownLatch latch = new CountDownLatch(1); - AtomicInteger callCount = new AtomicInteger(0); - doAnswer(invocation -> { - callCount.incrementAndGet(); - if (latch.await(5000, TimeUnit.SECONDS) == false) { - throw new Exception("Timed out waiting for delete task queuing to complete"); - } - return null; - }).when(blobContainer) - .listBlobsByPrefixInSortedOrder( - any(String.class), - any(int.class), - any(BlobContainer.BlobNameSortOrder.class), - any(ActionListener.class) - ); - - remoteClusterStateService.start(); - remoteClusterStateService.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS); - remoteClusterStateService.deleteStaleClusterMetadata("cluster-name", "cluster-uuid", RETAINED_MANIFESTS); - - latch.countDown(); - assertBusy(() -> assertEquals(1, callCount.get())); - } - - public void testRemoteCleanupTaskScheduled() { - AbstractAsyncTask cleanupTask = remoteClusterStateService.getStaleFileDeletionTask(); - assertNull(cleanupTask); - - remoteClusterStateService.start(); - assertNotNull(remoteClusterStateService.getStaleFileDeletionTask()); - assertTrue(remoteClusterStateService.getStaleFileDeletionTask().mustReschedule()); - assertEquals( - clusterSettings.get(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING), - remoteClusterStateService.getStaleFileDeletionTask().getInterval() - ); - assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); - assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isClosed()); - } - - public void testRemoteCleanupNotInitializedOnDataOnlyNode() { - String stateRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - "remote_store_repository" - ); - String stateRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - "remote_store_repository" - ); - Settings settings = Settings.builder() - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "remote_store_repository") - .putList(NODE_ROLES_SETTING.getKey(), "d") - .put(stateRepoTypeAttributeKey, FsRepository.TYPE) - .put(stateRepoSettingsAttributeKeyPrefix + "location", "randomRepoPath") - .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) - .build(); - ClusterSettings dataNodeClusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - remoteClusterStateService = new RemoteClusterStateService( - "test-node-id", - repositoriesServiceSupplier, - settings, - dataNodeClusterSettings, - () -> 0L, - threadPool - ); - remoteClusterStateService.start(); - assertNull(remoteClusterStateService.getStaleFileDeletionTask()); - } - public void testRemoteRoutingTableNotInitializedWhenDisabled() { assertNull(remoteClusterStateService.getRemoteRoutingTableService()); } @@ -1394,9 +1299,10 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { "test-node-id", repositoriesServiceSupplier, newSettings, - clusterSettings, + clusterService, () -> 0L, - threadPool + threadPool, + List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) ); assertNotNull(remoteClusterStateService.getRemoteRoutingTableService()); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java index 8ccc34e32f4b7..78e7b78513df3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java @@ -14,6 +14,7 @@ import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; +import org.opensearch.threadpool.TestThreadPool; import static org.mockito.Mockito.mock; @@ -26,7 +27,7 @@ public class RemoteIndexMetadataManagerTests extends OpenSearchTestCase { public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, new TestThreadPool("test")); } @After diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java index e539b382a5f3b..214d4baed2a02 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java @@ -46,6 +46,7 @@ import org.mockito.Mockito; +import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING; import static org.opensearch.index.remote.RemoteStoreEnums.PathType.FIXED; import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_INFIX; import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_PREFIX; @@ -276,7 +277,7 @@ public void testInterceptWithLatchAwaitTimeout() throws IOException { Settings settings = Settings.builder() .put(this.settings) - .put(RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) + .put(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) .build(); clusterSettings.applySettings(settings); SetOnce exceptionSetOnce = new SetOnce<>(); @@ -306,7 +307,7 @@ public void testInterceptWithInterruptedExceptionDuringLatchAwait() throws Excep remoteIndexPathUploader.start(); Settings settings = Settings.builder() .put(this.settings) - .put(RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(1)) + .put(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(1)) .build(); clusterSettings.applySettings(settings); SetOnce exceptionSetOnce = new SetOnce<>(); From 6d3390097998649b252da1c6aff5a3a7af0bf36b Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Tue, 21 May 2024 21:09:55 +0530 Subject: [PATCH 071/133] Enable Routing Table parallel read Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 62 +++++++++++++- .../remote/RemoteClusterStateService.java | 80 +++++++++++++++---- .../remote/RemoteClusterStateUtils.java | 4 +- .../RemoteRoutingTableServiceTests.java | 7 +- 4 files changed, 134 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 8a66e3ece8253..eabe4e129553d 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; +import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; @@ -24,10 +25,14 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStream; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStreamReader; import org.opensearch.index.remote.RemoteStoreUtils; @@ -36,6 +41,8 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; import java.io.*; import java.io.Closeable; @@ -46,6 +53,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -77,14 +85,16 @@ public class RemoteRoutingTableService implements Closeable { private final Supplier repositoriesService; private final ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; + private final ThreadPool threadPool; public RemoteRoutingTableService(Supplier repositoriesService, Settings settings, - ClusterSettings clusterSettings) { + ClusterSettings clusterSettings, ThreadPool threadPool) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; this.clusterSettings = clusterSettings; + this.threadPool = threadPool; } public List writeFullRoutingTable(ClusterState clusterState, String previousClusterUUID) { @@ -216,6 +226,39 @@ public RoutingTable getFullRoutingTable(long routingTableVersion, List getAsyncIndexMetadataReadAction( + String clusterName, + String clusterUUID, + String uploadedFilename, + Index index, + LatchedActionListener latchedActionListener) { + BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID)); + return () -> readAsync( + blobContainer, + uploadedFilename, + threadPool.executor(ThreadPool.Names.GENERIC), + ActionListener.wrap(response -> latchedActionListener.onResponse(new RemoteIndexRoutingResult(index.getName(), response.readIndexRoutingTable(index))), latchedActionListener::onFailure) + ); + } + + public void readAsync(BlobContainer blobContainer, String name, ExecutorService executorService, ActionListener listener) throws IOException { + executorService.execute(() -> { + try { + listener.onResponse(read(blobContainer, name)); + } catch (Exception e) { + listener.onFailure(e); + } + }); + } + + public IndexRoutingTableInputStreamReader read(BlobContainer blobContainer, String path) { + try { + new IndexRoutingTableInputStreamReader(blobContainer.readBlob(path)); + } catch (IOException e) { + logger.info("RoutingTable read failed with error: {}", e.toString()); + } + return null; + } private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) { } @@ -237,4 +280,21 @@ public void start() { blobStoreRepository = (BlobStoreRepository) repository; } + public static class RemoteIndexRoutingResult { + String indexName; + IndexRoutingTable indexRoutingTable; + + public RemoteIndexRoutingResult(String indexName, IndexRoutingTable indexRoutingTable) { + this.indexName = indexName; + this.indexRoutingTable = indexRoutingTable; + } + + public String getIndexName() { + return indexName; + } + + public IndexRoutingTable getIndexRoutingTable() { + return indexRoutingTable; + } + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index d21b38dc389fd..d1cf07e44e079 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -18,6 +18,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedRunnable; import org.opensearch.cluster.node.DiscoveryNodes; @@ -34,6 +35,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; @@ -183,7 +185,7 @@ public RemoteClusterStateService( if(isRemoteRoutingTableEnabled(settings)) { this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, - settings, clusterSettings); + settings, clusterSettings, threadPool); logger.info("REMOTE ROUTING ENABLED"); } else { logger.info("REMOTE ROUTING DISABLED"); @@ -821,12 +823,15 @@ private ClusterState readClusterStateInParallel( boolean readSettingsMetadata, boolean readTemplatesMetadata, boolean readDiscoveryNodes, - boolean readClusterBlocks + boolean readClusterBlocks, + List indicesRoutingToRead ) throws IOException { - int totalReadTasks = indicesToRead.size() + customToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + (readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0); + int totalReadTasks = indicesToRead.size() + customToRead.size() + indicesRoutingToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + (readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalReadTasks); List> asyncMetadataReadActions = new ArrayList<>(); List readResults = new ArrayList<>(); + List readIndexRoutingTableResults = new ArrayList<>(); + List indexRoutingExceptionList = Collections.synchronizedList(new ArrayList<>(indicesRoutingToRead.size())); List exceptionList = Collections.synchronizedList(new ArrayList<>(totalReadTasks)); LatchedActionListener listener = new LatchedActionListener<>( @@ -854,6 +859,32 @@ private ClusterState readClusterStateInParallel( ); } + LatchedActionListener routingTableLatchedActionListener = new LatchedActionListener<>( + ActionListener.wrap( + response -> { + logger.info("Successfully read cluster state component from remote"); + readIndexRoutingTableResults.add(response); + }, + ex -> { + logger.error("Failed to read cluster state from remote", ex); + indexRoutingExceptionList.add(ex); + } + ), + latch + ); + + for (UploadedIndexMetadata indexRouting: indicesRoutingToRead) { + asyncMetadataReadActions.add( + remoteRoutingTableService.getAsyncIndexMetadataReadAction( + clusterName, + clusterUUID, + indexRouting.getUploadedFilename(), + new Index(indexRouting.getIndexName(), indexRouting.getIndexUUID()), + routingTableLatchedActionListener + ) + ); + } + for (Map.Entry entry : customToRead.entrySet()) { asyncMetadataReadActions.add( remoteGlobalMetadataManager.getAsyncMetadataReadAction( @@ -960,12 +991,20 @@ private ClusterState readClusterStateInParallel( exceptionList.forEach(exception::addSuppressed); throw exception; } + + if (!indexRoutingExceptionList.isEmpty()) { + RemoteStateTransferException exception = new RemoteStateTransferException("Exception during reading routing table from remote"); + exceptionList.forEach(exception::addSuppressed); + throw exception; + } ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); AtomicReference discoveryNodesBuilder = new AtomicReference<>(DiscoveryNodes.builder()); Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); metadataBuilder.clusterUUID(manifest.getClusterUUID()); metadataBuilder.clusterUUIDCommitted(manifest.isClusterUUIDCommitted()); Map indexMetadataMap = new HashMap<>(); + Map indicesRouting = new HashMap<>(); + readResults.forEach(remoteReadResult -> { switch (remoteReadResult.getComponent()) { case INDEX_PATH_TOKEN: @@ -995,6 +1034,11 @@ private ClusterState readClusterStateInParallel( throw new IllegalStateException("Unknown component: " + remoteReadResult.getComponent()); } }); + + readIndexRoutingTableResults.forEach(remoteIndexRoutingResult -> { + indicesRouting.put(remoteIndexRoutingResult.getIndexName(), remoteIndexRoutingResult.getIndexRoutingTable()); + }); + metadataBuilder.indices(indexMetadataMap); if (readDiscoveryNodes) { clusterStateBuilder.nodes(discoveryNodesBuilder.get().localNodeId(localNodeId)); @@ -1002,6 +1046,7 @@ private ClusterState readClusterStateInParallel( return clusterStateBuilder.metadata(metadataBuilder) .version(manifest.getStateVersion()) .stateUUID(manifest.getStateUUID()) + .routingTable(new RoutingTable(manifest.getRoutingTableVersion(), indicesRouting)) .build(); } @@ -1019,7 +1064,8 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada manifest.getSettingsMetadata() != null, manifest.getTemplatesMetadata() != null, manifest.getDiscoveryNodesMetadata() != null, - manifest.getClusterBlocksMetadata() != null + manifest.getClusterBlocksMetadata() != null, + manifest.getIndicesRouting() ); } @@ -1031,6 +1077,13 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata assert uploadedIndexMetadataOptional.isPresent() == true; return uploadedIndexMetadataOptional.get(); }).collect(Collectors.toList()); + + List updatedIndexRouting = diff.getIndicesRoutingUpdated().stream().map(idx -> { + Optional uploadedIndexMetadataOptional = manifest.getIndicesRouting().stream().filter(idx2 -> idx2.getIndexName().equals(idx)).findFirst(); + assert uploadedIndexMetadataOptional.isPresent() == true; + return uploadedIndexMetadataOptional.get(); + }).collect(Collectors.toList()); + Map updatedCustomMetadata = new HashMap<>(); for (String customType : diff.getCustomMetadataUpdated().keySet()) { updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); @@ -1047,7 +1100,8 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata diff.isSettingsMetadataUpdated(), diff.isTemplatesMetadataUpdated(), diff.isDiscoveryNodesUpdated(), - diff.isClusterBlocksUpdated() + diff.isClusterBlocksUpdated(), + updatedIndexRouting ); ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); @@ -1065,23 +1119,21 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata metadataBuilder.removeCustom(customType); } - // Constructing Routing Table - if(remoteRoutingTableService!= null){ - RoutingTable routingTable = remoteRoutingTableService.getIncrementalRoutingTable(previousState, manifest); - return clusterStateBuilder. - stateUUID(manifest.getStateUUID()). - version(manifest.getStateVersion()). - metadata(metadataBuilder). - routingTable(routingTable). - build(); + HashMap indexRoutingTables = new HashMap<>(updatedClusterState.getRoutingTable().getIndicesRouting()); + + for(String indexName: diff.getIndicesRoutingDeleted()){ + indexRoutingTables.remove(indexName); } + RoutingTable routingTable = new RoutingTable(manifest.getRoutingTableVersion(), indexRoutingTables); return clusterStateBuilder. stateUUID(manifest.getStateUUID()). version(manifest.getStateVersion()). metadata(metadataBuilder). + routingTable(routingTable). build(); + } /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 3b71afd6026c3..dad69551435f7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -67,7 +67,7 @@ public RemoteStateTransferException(String errorDesc, Throwable cause) { } } - static class UploadedMetadataResults { + public static class UploadedMetadataResults { List uploadedIndexMetadata; Map uploadedCustomMetadataMap; ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata; @@ -105,7 +105,7 @@ public UploadedMetadataResults() { } } - static class RemoteReadResult { + public static class RemoteReadResult { ToXContent obj; String component; String componentName; diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 4026a36e76600..e5dff93b6074d 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -18,6 +18,7 @@ import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; import java.util.function.Supplier; @@ -55,7 +56,8 @@ public void setup() { remoteRoutingTableService = new RemoteRoutingTableService( repositoriesServiceSupplier, settings, - clusterSettings + clusterSettings, + new ThreadPool(settings) ); } @@ -73,7 +75,8 @@ public void testFailInitializationWhenRemoteRoutingDisabled() { () -> new RemoteRoutingTableService( repositoriesServiceSupplier, settings, - new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + new ThreadPool(settings) ) ); } From 99687b0333392841250a7b098d586665cfbd4229 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 22 May 2024 17:01:28 +0530 Subject: [PATCH 072/133] Create new remote state interfaces and add implementations Signed-off-by: Sooraj Sinha --- .../gradle/testclusters/OpenSearchNode.java | 9 +- .../remote/RemoteRoutingTableService.java | 9 +- .../remote/AbstractRemoteBlobStoreObject.java | 21 +- .../gateway/remote/BlobPathParameters.java | 2 - .../remote/ClusterMetadataManifest.java | 4 +- .../remote/ClusterStateDiffManifest.java | 7 + .../remote/RemoteClusterStateService.java | 166 ++++++++------- .../remote/RemoteClusterStateUtils.java | 17 +- .../remote/RemoteCoordinationMetadata.java | 117 +++++++++++ .../gateway/remote/RemoteCustomMetadata.java | 128 ++++++++++++ .../remote/RemoteGlobalMetadataManager.java | 197 ++++++++---------- .../gateway/remote/RemoteIndexMetadata.java | 88 ++++---- .../remote/RemoteIndexMetadataManager.java | 146 ++++--------- .../gateway/remote/RemoteObject.java | 5 +- .../gateway/remote/RemoteObjectBlobStore.java | 78 ++++--- .../gateway/remote/RemoteObjectManager.java | 39 ---- .../gateway/remote/RemoteObjectStore.java | 7 +- .../RemotePersistentSettingsMetadata.java | 115 ++++++++++ .../gateway/remote/RemoteStoreFactory.java | 19 -- .../remote/RemoteTemplatesMetadata.java | 114 ++++++++++ .../index/remote/RemoteIndexPathUploader.java | 3 + .../transfer/BlobStoreTransferService.java | 48 +++++ .../translog/transfer/TransferService.java | 2 + .../RemoteGlobalMetadataManagerTests.java | 4 +- .../RemoteIndexMetadataManagerTests.java | 6 +- .../remote/RemoteManifestManagerTests.java | 3 +- 26 files changed, 900 insertions(+), 454 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index 268de50340cbf..25df09ed9e4ac 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -1158,15 +1158,22 @@ private void createConfiguration() { if (nodeName != null) { baseConfig.put("node.name", nodeName); } - baseConfig.put("path.repo", confPathRepo.toAbsolutePath().toString()); + baseConfig.put("path.repo", "/Users/soosinha/os_data/repos"); baseConfig.put("path.data", confPathData.toAbsolutePath().toString()); baseConfig.put("path.logs", confPathLogs.toAbsolutePath().toString()); + baseConfig.put("cluster.remote_store.state.enabled", "true"); + baseConfig.put("path.shared_data", workingDir.resolve("sharedData").toString()); baseConfig.put("node.attr.testattr", "test"); if (StringUtils.isNotBlank(zone)) { baseConfig.put("cluster.routing.allocation.awareness.attributes", "zone"); baseConfig.put("node.attr.zone", zone); } + baseConfig.put("node.attr.remote_store.state.repository", "my-fs-repository"); + baseConfig.put("node.attr.remote_store.segment.repository", "my-fs-repository"); + baseConfig.put("node.attr.remote_store.translog.repository", "my-fs-repository"); + baseConfig.put("node.attr.remote_store.repository.my-fs-repository.type", "fs"); + baseConfig.put("node.attr.remote_store.repository.my-fs-repository.settings.location", "/Users/soosinha/os_data/repos/gradlerepo"); baseConfig.put("node.portsfile", "true"); baseConfig.put("http.port", httpPort); baseConfig.put("transport.port", transportPort); diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index eabe4e129553d..858583d2ddcdd 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -232,10 +232,12 @@ public CheckedRunnable getAsyncIndexMetadataReadAction( String uploadedFilename, Index index, LatchedActionListener latchedActionListener) { - BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID)); + BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(INDEX_ROUTING_PATH_TOKEN).add(index.getUUID())); + String[] fileNameTokens = uploadedFilename.split("/"); + String blobFileName = fileNameTokens[fileNameTokens.length -1]; return () -> readAsync( blobContainer, - uploadedFilename, + blobFileName, threadPool.executor(ThreadPool.Names.GENERIC), ActionListener.wrap(response -> latchedActionListener.onResponse(new RemoteIndexRoutingResult(index.getName(), response.readIndexRoutingTable(index))), latchedActionListener::onFailure) ); @@ -246,6 +248,7 @@ public void readAsync(BlobContainer blobContainer, String name, ExecutorService try { listener.onResponse(read(blobContainer, name)); } catch (Exception e) { + logger.error("routing table download failed : ", e); listener.onFailure(e); } }); @@ -253,7 +256,7 @@ public void readAsync(BlobContainer blobContainer, String name, ExecutorService public IndexRoutingTableInputStreamReader read(BlobContainer blobContainer, String path) { try { - new IndexRoutingTableInputStreamReader(blobContainer.readBlob(path)); + return new IndexRoutingTableInputStreamReader(blobContainer.readBlob(path)); } catch (IOException e) { logger.info("RoutingTable read failed with error: {}", e.toString()); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java index f6e3df100e5d1..e7f7466eb5a62 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java @@ -8,17 +8,24 @@ package org.opensearch.gateway.remote; -import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; + +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; public abstract class AbstractRemoteBlobStoreObject implements RemoteObject { - public abstract BlobContainer getBlobContainer(); public abstract BlobPathParameters getBlobPathParameters(); - public abstract String getBlobNameFormat(); - public abstract String getBlobName(); + public abstract String getFullBlobName(); + + public String getBlobFileName() { + if (getFullBlobName() == null) { + generateBlobFileName(); + } + String[] pathTokens = getFullBlobName().split(PATH_DELIMITER); + return getFullBlobName().split(PATH_DELIMITER)[pathTokens.length - 1]; + } public abstract String generateBlobFileName(); public abstract RemoteObjectStore getBackingStore(); - public abstract ChecksumBlobStoreFormat getChecksumBlobStoreFormat(); + public abstract UploadedMetadata getUploadedMetadata(); + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java b/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java index e6d101c27584a..4b47bd744ef1a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java +++ b/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java @@ -12,8 +12,6 @@ public class BlobPathParameters { - public static final String FILE_NAME_DELIMITER = "__"; - private List pathTokens; private String filePrefix; diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 56dde258886f5..1ace3a7b3b1f5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -998,8 +998,8 @@ public String getComponent() { } public String getUploadedFilename() { - String[] splitPath = uploadedFilename.split("/"); - return splitPath[splitPath.length - 1]; + // todo see how this change affects existing users + return uploadedFilename; } public String getIndexName() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 8a09252c99570..18518de4180df 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -13,6 +13,8 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParseException; @@ -253,6 +255,11 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw return builder.build(); } + @Override + public String toString() { + return Strings.toString(MediaTypeRegistry.JSON, this); + } + public List findRemovedIndices(Map indices, Map previousIndices) { List removedIndices = new ArrayList<>(); for (String index : previousIndices.keySet()) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index d1cf07e44e079..f79d7e1d5fa2b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -8,23 +8,65 @@ package org.opensearch.gateway.remote; +import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteReadResult; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT; +import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; -import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.CheckedRunnable; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; -import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobStore; @@ -40,6 +82,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -48,49 +91,6 @@ import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.function.LongSupplier; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteReadResult; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_DELIMITER; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; - /** * A Service which provides APIs to upload and download cluster metadata from remote store. * @@ -139,6 +139,7 @@ public class RemoteClusterStateService implements Closeable { private RemoteClusterStateAttributesManager remoteClusterStateAttributesManager; private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; + private BlobStoreTransferService blobStoreTransferService; private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " @@ -486,11 +487,11 @@ private UploadedMetadataResults writeMetadataInParallel( uploadTasks.put( SETTING_METADATA, remoteGlobalMetadataManager.getAsyncMetadataWriteAction( - clusterState, - SETTING_METADATA, - SETTINGS_METADATA_FORMAT, clusterState.metadata().persistentSettings(), - listener + clusterState.metadata().version(), + clusterState.metadata().clusterUUID(), + listener, + null ) ); } @@ -498,11 +499,11 @@ private UploadedMetadataResults writeMetadataInParallel( uploadTasks.put( COORDINATION_METADATA, remoteGlobalMetadataManager.getAsyncMetadataWriteAction( - clusterState, - COORDINATION_METADATA, - COORDINATION_METADATA_FORMAT, clusterState.metadata().coordinationMetadata(), - listener + clusterState.metadata().version(), + clusterState.metadata().clusterUUID(), + listener, + null ) ); } @@ -510,11 +511,11 @@ private UploadedMetadataResults writeMetadataInParallel( uploadTasks.put( TEMPLATES_METADATA, remoteGlobalMetadataManager.getAsyncMetadataWriteAction( - clusterState, - TEMPLATES_METADATA, - TEMPLATES_METADATA_FORMAT, clusterState.metadata().templatesMetadata(), - listener + clusterState.metadata().version(), + clusterState.metadata().clusterUUID(), + listener, + null ) ); } @@ -547,18 +548,18 @@ private UploadedMetadataResults writeMetadataInParallel( uploadTasks.put( customComponent, remoteGlobalMetadataManager.getAsyncMetadataWriteAction( - clusterState, - customComponent, - CUSTOM_METADATA_FORMAT, value, - listener + clusterState.metadata().version(), + clusterState.metadata().clusterUUID(), + listener, + key ) ); }); indexToUpload.forEach(indexMetadata -> { uploadTasks.put( indexMetadata.getIndexName(), - remoteIndexMetadataManager.getIndexMetadataAsyncAction(clusterState, indexMetadata, listener) + remoteIndexMetadataManager.getIndexMetadataAsyncAction(indexMetadata, clusterState.metadata().clusterUUID(), listener) ); }); @@ -761,8 +762,9 @@ public void start() { if(this.remoteRoutingTableService != null) { this.remoteRoutingTableService.start(); } - remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings, threadpool); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, threadpool); + String clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings).value(); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings,threadpool, getBlobStoreTransferService(), clusterName); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings,threadpool, clusterName, blobStoreRepository.getNamedXContentRegistry(), getBlobStoreTransferService()); remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(blobStoreRepository, threadpool); remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); remoteClusterStateCleanupManager.start(); @@ -851,7 +853,6 @@ private ClusterState readClusterStateInParallel( for (UploadedIndexMetadata indexMetadata : indicesToRead) { asyncMetadataReadActions.add( remoteIndexMetadataManager.getAsyncIndexMetadataReadAction( - clusterName, clusterUUID, indexMetadata.getUploadedFilename(), listener @@ -1085,8 +1086,10 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata }).collect(Collectors.toList()); Map updatedCustomMetadata = new HashMap<>(); - for (String customType : diff.getCustomMetadataUpdated().keySet()) { - updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); + if (diff.getCustomMetadataUpdated() != null) { + for (String customType : diff.getCustomMetadataUpdated().keySet()) { + updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); + } } ClusterState updatedClusterState = readClusterStateInParallel( previousState, @@ -1110,13 +1113,15 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata } if (diff.getCustomMetadataUpdated() != null) { for (String customType : diff.getCustomMetadataUpdated().keySet()) { - Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(clusterName, manifest.getClusterUUID(), + Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(manifest.getClusterUUID(), manifest.getCustomMetadataMap().get(customType).getUploadedFilename(), customType); metadataBuilder.putCustom(customType, custom); } } - for (String customType : diff.getCustomMetadataDeleted()) { - metadataBuilder.removeCustom(customType); + if (diff.getCustomMetadataDeleted() != null) { + for (String customType : diff.getCustomMetadataDeleted()) { + metadataBuilder.removeCustom(customType); + } } HashMap indexRoutingTables = new HashMap<>(updatedClusterState.getRoutingTable().getIndicesRouting()); @@ -1171,6 +1176,13 @@ public RemoteClusterStateCleanupManager getCleanupManager() { return remoteClusterStateCleanupManager; } + private BlobStoreTransferService getBlobStoreTransferService() { + if (blobStoreTransferService == null) { + blobStoreTransferService = new BlobStoreTransferService(getBlobStore(), threadpool); + } + return blobStoreTransferService; + } + Set getAllClusterUUIDs(String clusterName) throws IOException { Map clusterUUIDMetadata = clusterUUIDContainer(blobStoreRepository, clusterName).children(); if (clusterUUIDMetadata == null) { @@ -1271,21 +1283,21 @@ private boolean isMetadataEqual(ClusterMetadataManifest first, ClusterMetadataMa } final Map secondIndices = second.getIndices() .stream() - .collect(Collectors.toMap(md -> md.getIndexName(), Function.identity())); + .collect(Collectors.toMap(UploadedIndexMetadata::getIndexName, Function.identity())); for (UploadedIndexMetadata uploadedIndexMetadata : first.getIndices()) { final IndexMetadata firstIndexMetadata = remoteIndexMetadataManager.getIndexMetadata( - clusterName, + uploadedIndexMetadata, first.getClusterUUID(), - uploadedIndexMetadata + first.getCodecVersion() ); final UploadedIndexMetadata secondUploadedIndexMetadata = secondIndices.get(uploadedIndexMetadata.getIndexName()); if (secondUploadedIndexMetadata == null) { return false; } final IndexMetadata secondIndexMetadata = remoteIndexMetadataManager.getIndexMetadata( - clusterName, + secondUploadedIndexMetadata, second.getClusterUUID(), - secondUploadedIndexMetadata + second.getCodecVersion() ); if (firstIndexMetadata.equals(secondIndexMetadata) == false) { return false; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index dad69551435f7..abedd23c1e742 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote; +import java.util.Locale; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; @@ -23,18 +24,15 @@ public class RemoteClusterStateUtils { public static final String METADATA_NAME_FORMAT = "%s.dat"; + public static final String METADATA_NAME_PLAIN_FORMAT = "%s"; public static final String METADATA_FILE_PREFIX = "metadata"; public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; public static final String DELIMITER = "__"; + public static final String PATH_DELIMITER = "/"; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. - public static final ToXContent.Params FORMAT_PARAMS; - static { - Map params = new HashMap<>(1); - params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY); - FORMAT_PARAMS = new ToXContent.MapParams(params); - } + public static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY)); public static BlobPath getCusterMetadataBasePath(BlobStoreRepository blobStoreRepository, String clusterName, String clusterUUID) { return blobStoreRepository.basePath().add(encodeString(clusterName)).add(CLUSTER_STATE_PATH_TOKEN).add(clusterUUID); @@ -44,6 +42,13 @@ public static String encodeString(String content) { return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8)); } + public static String getFormattedFileName(String fileName, int codecVersion) { + if (codecVersion < ClusterMetadataManifest.CODEC_V3) { + return String.format(Locale.ROOT, METADATA_NAME_FORMAT, fileName); + } + return fileName; + } + static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepository, String clusterName) { return blobStoreRepository.blobStore() .blobContainer( diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java new file mode 100644 index 0000000000000..633a74b52b85a --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.io.Streams; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class RemoteCoordinationMetadata extends AbstractRemoteBlobStoreObject { + + public static final String COORDINATION_METADATA = "coordination"; + public static final ChecksumBlobStoreFormat COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "coordination", + METADATA_NAME_FORMAT, + CoordinationMetadata::fromXContent + ); + + private CoordinationMetadata coordinationMetadata; + private long metadataVersion; + private final RemoteObjectStore backingStore; + private String blobName; + private final NamedXContentRegistry xContentRegistry; + private final String clusterUUID; + + public RemoteCoordinationMetadata(CoordinationMetadata coordinationMetadata, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, + NamedXContentRegistry xContentRegistry) { + this.coordinationMetadata = coordinationMetadata; + this.metadataVersion = metadataVersion; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + } + + public RemoteCoordinationMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + this.blobName = blobName; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("global-metadata"), COORDINATION_METADATA); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + assert backingStore instanceof RemoteObjectBlobStore; + RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; + // setting the full blob path with name for future access + this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public CoordinationMetadata get() { + return coordinationMetadata; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + @Override + public RemoteObjectStore getBackingStore() { + return backingStore; + } + + @Override + public InputStream serialize() throws IOException { + return COORDINATION_METADATA_FORMAT.serialize(coordinationMetadata, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public CoordinationMetadata deserialize(InputStream inputStream) throws IOException { + return COORDINATION_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(COORDINATION_METADATA, blobName); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java new file mode 100644 index 0000000000000..7220f11b8de63 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.Metadata.Custom; +import org.opensearch.common.io.Streams; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class RemoteCustomMetadata extends AbstractRemoteBlobStoreObject{ + + public static final String CUSTOM_METADATA = "custom"; + public static final String CUSTOM_DELIMITER = "--"; + + public final ChecksumBlobStoreFormat customBlobStoreFormat; + + private Custom custom; + private String customType; + private long metadataVersion; + private final RemoteObjectStore backingStore; + private String blobName; + private final NamedXContentRegistry xContentRegistry; + private final String clusterUUID; + + public RemoteCustomMetadata(Custom custom, String customType, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, + NamedXContentRegistry xContentRegistry) { + this.custom = custom; + this.customType = customType; + this.metadataVersion = metadataVersion; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + this.customBlobStoreFormat = new ChecksumBlobStoreFormat<>( + "custom", + METADATA_NAME_FORMAT, + (parser -> Metadata.Custom.fromXContent(parser, customType)) + ); + } + + public RemoteCustomMetadata(String blobName, String customType, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + this.blobName = blobName; + this.customType = customType; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + this.customBlobStoreFormat = new ChecksumBlobStoreFormat<>( + "custom", + METADATA_NAME_FORMAT, + (parser -> Metadata.Custom.fromXContent(parser, customType)) + ); + } + + @Override + public BlobPathParameters getBlobPathParameters() { + String prefix = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, customType); + return new BlobPathParameters(List.of("global-metadata"), prefix); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + assert backingStore instanceof RemoteObjectBlobStore; + RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; + // setting the full blob path with name for future access + this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public Custom get() { + return custom; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + @Override + public RemoteObjectStore getBackingStore() { + return backingStore; + } + + @Override + public InputStream serialize() throws IOException { + return customBlobStoreFormat.serialize(custom, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public Custom deserialize(InputStream inputStream) throws IOException { + return customBlobStoreFormat.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, customType), blobName); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 76346376255b0..ad7cedefb965e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -8,10 +8,22 @@ package org.opensearch.gateway.remote; +import static java.util.Objects.requireNonNull; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.blobstore.BlobContainer; @@ -21,30 +33,13 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CountDownLatch; - -import static java.util.Objects.requireNonNull; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; - public class RemoteGlobalMetadataManager { + public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; public static final String COORDINATION_METADATA = "coordination"; public static final String SETTING_METADATA = "settings"; @@ -85,54 +80,47 @@ public class RemoteGlobalMetadataManager { TemplatesMetadata::fromXContent ); - public static final ChecksumBlobStoreFormat CUSTOM_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "custom", - METADATA_NAME_FORMAT, - Metadata.Custom::fromXContent - ); public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; private final BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; private volatile TimeValue globalMetadataUploadTimeout; + private final RemoteObjectBlobStore coordinationMetadataStore; + private final RemoteObjectBlobStore persistentSettingsBlobStore; + private final RemoteObjectBlobStore templatesMetadataBlobStore; + private final RemoteObjectBlobStore customMetadataBlobStore; - RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool) { + RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, BlobStoreTransferService blobStoreTransferService, + String clusterName) { this.blobStoreRepository = blobStoreRepository; this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); this.threadPool = threadPool; clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); + this.coordinationMetadataStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); + this.persistentSettingsBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); + this.templatesMetadataBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); + this.customMetadataBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); } /** * Allows async upload of Metadata components to remote */ CheckedRunnable getAsyncMetadataWriteAction( - ClusterState clusterState, - String component, - ChecksumBlobStoreFormat componentMetadataBlobStore, - ToXContent componentMetadata, - LatchedActionListener latchedActionListener + Object objectToUpload, + long metadataVersion, + String clusterUUID, + LatchedActionListener latchedActionListener, + String customType ) { - final BlobContainer globalMetadataContainer = globalMetadataContainer( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() - ); - final String componentMetadataFilename = metadataAttributeFileName(component, clusterState.metadata().version()); + AbstractRemoteBlobStoreObject remoteBlobStoreObject = getRemoteObject(objectToUpload, metadataVersion, clusterUUID, customType); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( - new ClusterMetadataManifest.UploadedMetadataAttribute(component, componentMetadataFilename) + remoteBlobStoreObject.getUploadedMetadata() ), - ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, ex)) - ); - return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority( - componentMetadata, - globalMetadataContainer, - componentMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS + ex -> latchedActionListener.onFailure(new RemoteStateTransferException("Upload failed", ex)) ); + return remoteBlobStoreObject.getBackingStore().writeAsync(remoteBlobStoreObject, completionListener); } CheckedRunnable getAsyncMetadataReadAction( @@ -144,14 +132,28 @@ CheckedRunnable getAsyncMetadataReadAction( ChecksumBlobStoreFormat componentBlobStore, LatchedActionListener listener ) { - String[] splitName = uploadFilename.split("/"); - return () -> componentBlobStore.readAsync( - globalMetadataContainer(clusterName, clusterUUID), - splitName[splitName.length - 1], - blobStoreRepository.getNamedXContentRegistry(), - threadPool.executor(ThreadPool.Names.GENERIC), - ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure) - ); + AbstractRemoteBlobStoreObject remoteBlobStoreObject; + if (component.equals(COORDINATION_METADATA)) { + remoteBlobStoreObject = new RemoteCoordinationMetadata(uploadFilename, clusterUUID, coordinationMetadataStore, blobStoreRepository.getNamedXContentRegistry()); + } else if (component.equals(TEMPLATES_METADATA)) { + remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, templatesMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + } else if (component.equals(SETTING_METADATA)) { + remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, persistentSettingsBlobStore, blobStoreRepository.getNamedXContentRegistry()); + } else if (component.equals(CUSTOM_METADATA)) { + remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, customMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + } else { + throw new RemoteStateTransferException("Unknown component " + componentName); + } + ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); + return () -> remoteBlobStoreObject.getBackingStore().readAsync(remoteBlobStoreObject, actionListener); +// String[] splitName = uploadFilename.split("/"); +// return () -> componentBlobStore.readAsync( +// globalMetadataContainer(clusterName, clusterUUID), +// splitName[splitName.length - 1], +// blobStoreRepository.getNamedXContentRegistry(), +// threadPool.executor(ThreadPool.Names.GENERIC), +// ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure) +// ); } Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { @@ -167,17 +169,14 @@ Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetada ); } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { CoordinationMetadata coordinationMetadata = getCoordinationMetadata( - clusterName, clusterUUID, clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() ); Settings settingsMetadata = getSettingsMetadata( - clusterName, clusterUUID, clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() ); TemplatesMetadata templatesMetadata = getTemplatesMetadata( - clusterName, clusterUUID, clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() ); @@ -193,7 +192,7 @@ Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetada .forEach( (key, value) -> builder.putCustom( key, - getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key) + getCustomsMetadata(clusterUUID, value.getUploadedFilename(), key) ) ); return builder.build(); @@ -208,16 +207,13 @@ Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetada } } - public CoordinationMetadata getCoordinationMetadata(String clusterName, String clusterUUID, String coordinationMetadataFileName) { + public CoordinationMetadata getCoordinationMetadata(String clusterUUID, String coordinationMetadataFileName) { try { // Fetch Coordination metadata if (coordinationMetadataFileName != null) { - String[] splitPath = coordinationMetadataFileName.split("/"); - return COORDINATION_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); + RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(coordinationMetadataFileName, clusterUUID, + coordinationMetadataStore, blobStoreRepository.getNamedXContentRegistry()); + return remoteCoordinationMetadata.getBackingStore().read(remoteCoordinationMetadata); } else { return CoordinationMetadata.EMPTY_METADATA; } @@ -229,16 +225,13 @@ public CoordinationMetadata getCoordinationMetadata(String clusterName, String c } } - public Settings getSettingsMetadata(String clusterName, String clusterUUID, String settingsMetadataFileName) { + public Settings getSettingsMetadata(String clusterUUID, String settingsMetadataFileName) { try { // Fetch Settings metadata if (settingsMetadataFileName != null) { - String[] splitPath = settingsMetadataFileName.split("/"); - return SETTINGS_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); + RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(settingsMetadataFileName, clusterUUID, + persistentSettingsBlobStore, blobStoreRepository.getNamedXContentRegistry()); + return remotePersistentSettingsMetadata.getBackingStore().read(remotePersistentSettingsMetadata); } else { return Settings.EMPTY; } @@ -250,16 +243,13 @@ public Settings getSettingsMetadata(String clusterName, String clusterUUID, Stri } } - public TemplatesMetadata getTemplatesMetadata(String clusterName, String clusterUUID, String templatesMetadataFileName) { + public TemplatesMetadata getTemplatesMetadata(String clusterUUID, String templatesMetadataFileName) { try { // Fetch Templates metadata if (templatesMetadataFileName != null) { - String[] splitPath = templatesMetadataFileName.split("/"); - return TEMPLATES_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); + RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(templatesMetadataFileName, clusterUUID, + templatesMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + return remoteTemplatesMetadata.getBackingStore().read(remoteTemplatesMetadata); } else { return TemplatesMetadata.EMPTY_METADATA; } @@ -271,21 +261,12 @@ public TemplatesMetadata getTemplatesMetadata(String clusterName, String cluster } } - public Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) { + public Metadata.Custom getCustomsMetadata(String clusterUUID, String customMetadataFileName, String custom) { requireNonNull(customMetadataFileName); try { // Fetch Custom metadata - String[] splitPath = customMetadataFileName.split("/"); - ChecksumBlobStoreFormat customChecksumBlobStoreFormat = new ChecksumBlobStoreFormat<>( - "custom", - METADATA_NAME_FORMAT, - (parser -> Metadata.Custom.fromXContent(parser, custom)) - ); - return customChecksumBlobStoreFormat.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); + RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, customMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + return remoteCustomMetadata.getBackingStore().read(remoteCustomMetadata); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), @@ -319,34 +300,26 @@ Map getUpdatedCustoms(ClusterState currentState, Cluste return updatedCustom; } + private AbstractRemoteBlobStoreObject getRemoteObject(Object object, long metadataVersion, String clusterUUID, String customType) { + if (object instanceof CoordinationMetadata) { + return new RemoteCoordinationMetadata((CoordinationMetadata) object, metadataVersion, clusterUUID, coordinationMetadataStore, + blobStoreRepository.getNamedXContentRegistry()); + } else if (object instanceof Settings) { + return new RemotePersistentSettingsMetadata((Settings) object, metadataVersion, clusterUUID, persistentSettingsBlobStore, blobStoreRepository.getNamedXContentRegistry()); + } else if (object instanceof TemplatesMetadata) { + return new RemoteTemplatesMetadata((TemplatesMetadata) object, metadataVersion, clusterUUID, templatesMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + } else if (object instanceof Custom) { + return new RemoteCustomMetadata((Custom) object, customType ,metadataVersion, clusterUUID, customMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + } + throw new RemoteStateTransferException("Remote object cannot be created for " + object.getClass()); + } + private BlobContainer globalMetadataContainer(String clusterName, String clusterUUID) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/ return blobStoreRepository.blobStore() .blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(GLOBAL_METADATA_PATH_TOKEN)); } - private static String globalMetadataFileName(Metadata metadata) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/metadata______ - return String.join( - DELIMITER, - METADATA_FILE_PREFIX, - RemoteStoreUtils.invertLong(metadata.version()), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) - ); - } - - private static String metadataAttributeFileName(String componentPrefix, Long metadataVersion) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ - return String.join( - DELIMITER, - componentPrefix, - RemoteStoreUtils.invertLong(metadataVersion), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) - ); - } - boolean isGlobalMetadataEqual(ClusterMetadataManifest first, ClusterMetadataManifest second, String clusterName) { Metadata secondGlobalMetadata = getGlobalMetadata(clusterName, second.getClusterUUID(), second); Metadata firstGlobalMetadata = getGlobalMetadata(clusterName, first.getClusterUUID(), first); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java index 32f009e931052..1543dc0ed11a7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java @@ -9,50 +9,47 @@ package org.opensearch.gateway.remote; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_PLAIN_FORMAT; + import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.common.io.Streams; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; public class RemoteIndexMetadata extends AbstractRemoteBlobStoreObject { - - public static final String DELIMITER = "__"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( "index-metadata", - RemoteClusterStateUtils.METADATA_NAME_FORMAT, + METADATA_NAME_PLAIN_FORMAT, IndexMetadata::fromXContent ); - public static final ToXContent.Params FORMAT_PARAMS; - - static { - Map params = new HashMap<>(1); - params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY); - FORMAT_PARAMS = new ToXContent.MapParams(params); - } - private IndexMetadata indexMetadata; private final RemoteObjectStore backingStore; private String blobName; + private final NamedXContentRegistry xContentRegistry; + private final String clusterUUID; - public RemoteIndexMetadata(IndexMetadata indexMetadata, RemoteObjectStore backingStore) { + public RemoteIndexMetadata(IndexMetadata indexMetadata, String clusterUUID, RemoteObjectBlobStore backingStore, + NamedXContentRegistry xContentRegistry) { this.indexMetadata = indexMetadata; this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; } - public RemoteIndexMetadata(String blobName, RemoteObjectStore backingStore) { + public RemoteIndexMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { this.blobName = blobName; this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; } @Override @@ -61,40 +58,35 @@ public IndexMetadata get() { } @Override - public BlobContainer getBlobContainer() { - return null; - } - - @Override - public BlobPathParameters getBlobPathParameters() { - return new BlobPathParameters(List.of("index"), "metadata"); - } - - @Override - public ChecksumBlobStoreFormat getChecksumBlobStoreFormat() { - return INDEX_METADATA_FORMAT; + public String clusterUUID() { + return clusterUUID; } @Override - public String getBlobNameFormat() { - return null; + public String getFullBlobName() { + return blobName; } @Override - public String getBlobName() { - return blobName; + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("index"), "metadata"); } @Override public String generateBlobFileName() { - return String.join( - DELIMITER, + String blobFileName = String.join( + RemoteClusterStateUtils.DELIMITER, getBlobPathParameters().getFilePrefix(), RemoteStoreUtils.invertLong(indexMetadata.getVersion()), RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last + String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during reads we read last // place to determine codec version. ); + assert backingStore instanceof RemoteObjectBlobStore; + RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; + // setting the full blob path with name for future access + this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + return blobFileName; } @Override @@ -103,12 +95,26 @@ public RemoteObjectStore getBackingStore() { } @Override - public BytesReference serialize() throws IOException { - return INDEX_METADATA_FORMAT.serialize(indexMetadata, generateBlobFileName(), getBackingStore().getCompressor(), FORMAT_PARAMS); + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedIndexMetadata(indexMetadata.getIndexName(), indexMetadata.getIndexUUID(), blobName); + } + + @Override + public InputStream serialize() throws IOException { + return INDEX_METADATA_FORMAT.serialize(indexMetadata, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); } @Override public IndexMetadata deserialize(InputStream inputStream) throws IOException { - return null; + // Blob name parameter is redundant + return INDEX_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + } + + @Override + public String toString() { + return blobName + clusterUUID; } + + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index 3d50b9d8453fa..509cffce86b2c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -8,35 +8,26 @@ package org.opensearch.gateway.remote; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; + +import java.io.IOException; +import java.util.Locale; import org.opensearch.action.LatchedActionListener; -import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; -import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; -import java.io.IOException; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; - -import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; - public class RemoteIndexMetadataManager { + public static final TimeValue INDEX_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); public static final Setting INDEX_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( @@ -56,91 +47,75 @@ public class RemoteIndexMetadataManager { private final BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; + private final NamedXContentRegistry namedXContentRegistry; + private final RemoteObjectBlobStore remoteIndexMetadataBlobStore; + private volatile TimeValue indexMetadataUploadTimeout; - public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool) { + public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, String clusterName, + NamedXContentRegistry namedXContentRegistry, BlobStoreTransferService blobStoreTransferService) { this.blobStoreRepository = blobStoreRepository; this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); this.threadPool = threadPool; clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); + this.remoteIndexMetadataBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); + this.namedXContentRegistry = namedXContentRegistry; } /** * Allows async Upload of IndexMetadata to remote * - * @param clusterState current ClusterState * @param indexMetadata {@link IndexMetadata} to upload * @param latchedActionListener listener to respond back on after upload finishes */ - CheckedRunnable getIndexMetadataAsyncAction( - ClusterState clusterState, - IndexMetadata indexMetadata, - LatchedActionListener latchedActionListener - ) { - final BlobContainer indexMetadataContainer = indexMetadataContainer( - clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID(), - indexMetadata.getIndexUUID() - ); - final String indexMetadataFilename = indexMetadataFileName(indexMetadata); + CheckedRunnable getIndexMetadataAsyncAction(IndexMetadata indexMetadata, String clusterUUID, + LatchedActionListener latchedActionListener) { + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(indexMetadata, clusterUUID, remoteIndexMetadataBlobStore, namedXContentRegistry); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( - new ClusterMetadataManifest.UploadedIndexMetadata( - indexMetadata.getIndex().getName(), - indexMetadata.getIndexUUID(), - indexMetadataContainer.path().buildAsString() + indexMetadataFilename - ) + remoteIndexMetadata.getUploadedMetadata() ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) ); - - return () -> INDEX_METADATA_FORMAT.writeAsyncWithUrgentPriority( - indexMetadata, - indexMetadataContainer, - indexMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); + return remoteIndexMetadataBlobStore.writeAsync(remoteIndexMetadata, completionListener); } CheckedRunnable getAsyncIndexMetadataReadAction( - String clusterName, String clusterUUID, String uploadedFilename, LatchedActionListener latchedActionListener ) { - String[] splitPath = uploadedFilename.split("/"); - return () -> INDEX_METADATA_FORMAT.readAsync( - indexMetadataContainer(clusterName, clusterUUID, splitPath[0]), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry(), - threadPool.executor(ThreadPool.Names.GENERIC), - ActionListener.wrap(response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, splitPath[0])), latchedActionListener::onFailure) - ); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, remoteIndexMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + ActionListener actionListener = ActionListener.wrap( + //todo change dummy + response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, "dummy")), + latchedActionListener::onFailure); + return () -> remoteIndexMetadataBlobStore.readAsync(remoteIndexMetadata, actionListener); +// String[] splitPath = uploadedFilename.split("/"); +// return () -> INDEX_METADATA_FORMAT.readAsync( +// indexMetadataContainer(clusterName, clusterUUID, splitPath[0]), +// splitPath[splitPath.length - 1], +// blobStoreRepository.getNamedXContentRegistry(), +// threadPool.executor(ThreadPool.Names.GENERIC), +// ActionListener.wrap( +// response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, "dummy")), +// latchedActionListener::onFailure) +// ); } /** * Fetch index metadata from remote cluster state * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster * @param uploadedIndexMetadata {@link ClusterMetadataManifest.UploadedIndexMetadata} contains details about remote location of index metadata * @return {@link IndexMetadata} */ IndexMetadata getIndexMetadata( - String clusterName, - String clusterUUID, - ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata + ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata, String clusterUUID, int manifestCodecVersion ) { - BlobContainer blobContainer = indexMetadataContainer(clusterName, clusterUUID, uploadedIndexMetadata.getIndexUUID()); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(RemoteClusterStateUtils.getFormattedFileName( + uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, remoteIndexMetadataBlobStore, namedXContentRegistry); try { - String[] splitPath = uploadedIndexMetadata.getUploadedFilename().split("/"); - return INDEX_METADATA_FORMAT.read( - blobContainer, - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); + return remoteIndexMetadataBlobStore.read(remoteIndexMetadata); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading IndexMetadata - %s", uploadedIndexMetadata.getUploadedFilename()), @@ -149,35 +124,6 @@ IndexMetadata getIndexMetadata( } } - /** - * Fetch latest index metadata from remote cluster state - * - * @param clusterUUID uuid of cluster state to refer to in remote - * @param clusterName name of the cluster - * @param clusterMetadataManifest manifest file of cluster - * @return {@code Map} latest IndexUUID to IndexMetadata map - */ - Map getIndexMetadataMap( - String clusterName, - String clusterUUID, - ClusterMetadataManifest clusterMetadataManifest - ) { - assert Objects.equals(clusterUUID, clusterMetadataManifest.getClusterUUID()) - : "Corrupt ClusterMetadataManifest found. Cluster UUID mismatch."; - Map remoteIndexMetadata = new HashMap<>(); - for (ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata : clusterMetadataManifest.getIndices()) { - IndexMetadata indexMetadata = getIndexMetadata(clusterName, clusterUUID, uploadedIndexMetadata); - remoteIndexMetadata.put(uploadedIndexMetadata.getIndexUUID(), indexMetadata); - } - return remoteIndexMetadata; - } - - private BlobContainer indexMetadataContainer(String clusterName, String clusterUUID, String indexUUID) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX - return blobStoreRepository.blobStore() - .blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(INDEX_PATH_TOKEN).add(indexUUID)); - } - public TimeValue getIndexMetadataUploadTimeout() { return this.indexMetadataUploadTimeout; } @@ -186,16 +132,4 @@ private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeo this.indexMetadataUploadTimeout = newIndexMetadataUploadTimeout; } - static String indexMetadataFileName(IndexMetadata indexMetadata) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index//metadata______ - return String.join( - DELIMITER, - METADATA_FILE_PREFIX, - RemoteStoreUtils.invertLong(indexMetadata.getVersion()), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last - // place to determine codec version. - ); - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java index 46b706583809b..e08d76ad3a6b5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java @@ -15,8 +15,9 @@ public interface RemoteObject { public T get(); + public String clusterUUID(); public RemoteObjectStore getBackingStore(); - public BytesReference serialize() throws IOException; + public InputStream serialize() throws IOException; + public T deserialize(InputStream inputStream) throws IOException; - T deserialize(InputStream inputStream) throws IOException; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java index 9a84243ae0006..a83d1178c7b20 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java @@ -10,60 +10,56 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.function.Function; +import java.util.Arrays; +import java.util.concurrent.ExecutorService; import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.compress.Compressor; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; public class RemoteObjectBlobStore implements RemoteObjectStore { + + private static final String PATH_DELIMITER = "/"; + + private final ExecutorService executorService; + private final BlobStoreTransferService transferService; private final BlobStoreRepository blobStoreRepository; private final String clusterName; - private final String clusterUUID; - public RemoteObjectBlobStore(BlobStoreRepository remoteStoreFactory, String clusterName, String clusterUUID) { + public RemoteObjectBlobStore(ExecutorService executorService, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository remoteStoreFactory, String clusterName) { + this.executorService = executorService; + this.transferService = blobStoreTransferService; this.blobStoreRepository = remoteStoreFactory; this.clusterName = clusterName; - this.clusterUUID = clusterUUID; - } - - @Override - public void write(RemoteObject remoteObject) throws IOException { - AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; - BytesReference bytesReference = abstractRemoteBlobStoreObject.serialize(); - abstractRemoteBlobStoreObject.getBlobContainer().writeBlob(abstractRemoteBlobStoreObject.generateBlobFileName(), bytesReference.streamInput(), bytesReference.length(), false); } @Override public CheckedRunnable writeAsync(RemoteObject remoteObject, ActionListener listener) { return () -> { AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; - BytesReference bytesReference = abstractRemoteBlobStoreObject.serialize(); - abstractRemoteBlobStoreObject.getBlobContainer().writeBlob(abstractRemoteBlobStoreObject.generateBlobFileName(), bytesReference.streamInput(), bytesReference.length(), false); + InputStream inputStream = abstractRemoteBlobStoreObject.serialize(); + transferService.uploadBlob(inputStream, getBlobPathForUpload(remoteObject), abstractRemoteBlobStoreObject.getBlobFileName(), WritePriority.URGENT, + listener); }; } @Override public T read(RemoteObject remoteObject) throws IOException { AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; - return abstractRemoteBlobStoreObject.deserialize(getBlobContainer(abstractRemoteBlobStoreObject.getBlobPathParameters().getPathTokens()).readBlob(abstractRemoteBlobStoreObject.getBlobName())); + return remoteObject.deserialize( + transferService.downloadBlob(getBlobPathForDownload(remoteObject), abstractRemoteBlobStoreObject.getBlobFileName())); } @Override - public CompletableFuture readAsync(RemoteObject remoteObject){ - return CompletableFuture.supplyAsync(() -> { - AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; + public void readAsync(RemoteObject remoteObject, ActionListener listener) { + executorService.execute(() -> { try { - return abstractRemoteBlobStoreObject.deserialize(getBlobContainer(abstractRemoteBlobStoreObject.getBlobPathParameters().getPathTokens()).readBlob(abstractRemoteBlobStoreObject.getBlobName())); - } catch (IOException e) { - throw new RuntimeException(e); + listener.onResponse(read(remoteObject)); + } catch (Exception e) { + listener.onFailure(e); } }); } @@ -73,16 +69,36 @@ public Compressor getCompressor() { return blobStoreRepository.getCompressor(); } - private BlobContainer getBlobContainer(List pathTokens) { - BlobPath blobPath = blobStoreRepository.basePath().add(encodeString(clusterName)).add("cluster-state").add(clusterUUID); + public BlobPath getBlobPathForUpload(RemoteObject remoteObject) { + BlobPath blobPath = blobStoreRepository.basePath().add(RemoteClusterStateUtils.encodeString(clusterName)).add("cluster-state").add(remoteObject.clusterUUID()); + AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; + for (String token : abstractRemoteBlobStoreObject.getBlobPathParameters().getPathTokens()) { + blobPath = blobPath.add(token); + } + return blobPath; + } + + public BlobPath getBlobPathForDownload(RemoteObject remoteObject) { + AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; + String[] pathTokens = extractBlobPathTokens(abstractRemoteBlobStoreObject.getFullBlobName()); + BlobPath blobPath = blobStoreRepository.basePath(); for (String token : pathTokens) { blobPath = blobPath.add(token); } - return blobStoreRepository.blobStore().blobContainer(blobPath); + return blobPath; + } + + private static String extractBlobFileName(String blobName) { + // todo add handling for *.dat extension + assert blobName != null; + String[] blobNameTokens = blobName.split(PATH_DELIMITER); + return blobNameTokens[blobNameTokens.length - 1]; } - public static String encodeString(String content) { - return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8)); + private static String[] extractBlobPathTokens(String blobName) { + String[] blobNameTokens = blobName.split(PATH_DELIMITER); + return Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1); } + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java deleted file mode 100644 index acacaf1ada6da..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectManager.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.List; -import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.core.xcontent.ToXContent; - -public class RemoteObjectManager { - - private final RemoteStoreFactory remoteStoreFactory; - private final String clusterName; - private final String clusterUUID; - - public RemoteObjectManager(RemoteStoreFactory remoteStoreFactory, String clusterName, String clusterUUID) { - this.remoteStoreFactory = remoteStoreFactory; - this.clusterName = clusterName; - this.clusterUUID = clusterUUID; - } - - public void write(RemoteObject remoteObject) throws IOException { - remoteObject.getBackingStore().write(remoteObject); - } - - public T read(RemoteObject remoteObject) throws IOException { - return remoteObject.getBackingStore().read(remoteObject); - } - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java index 451fbe121d48e..54ac83d5d83c8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java @@ -9,7 +9,9 @@ package org.opensearch.gateway.remote; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.CompletableFuture; +import java.util.function.Function; import org.opensearch.common.CheckedRunnable; import org.opensearch.core.action.ActionListener; import org.opensearch.core.compress.Compressor; @@ -17,10 +19,9 @@ public interface RemoteObjectStore { - public void write(RemoteObject remoteObject) throws IOException; public CheckedRunnable writeAsync(RemoteObject remoteObject, ActionListener listener); - public T read(RemoteObject remoteObject) throws IOException; - public CompletableFuture readAsync(RemoteObject remoteObject); + public T read(RemoteObject remoteObject) throws IOException; + public void readAsync(RemoteObject remoteObject, ActionListener listener); public Compressor getCompressor(); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java new file mode 100644 index 0000000000000..3a9e515a03ad0 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.common.io.Streams; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobStoreObject { + + public static final String SETTING_METADATA = "settings"; + + public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "settings", + METADATA_NAME_FORMAT, + Settings::fromXContent + ); + + private Settings persistentSettings; + private long metadataVersion; + private final RemoteObjectStore backingStore; + private String blobName; + private final NamedXContentRegistry xContentRegistry; + private final String clusterUUID; + + public RemotePersistentSettingsMetadata(Settings settings, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, + NamedXContentRegistry xContentRegistry) { + this.persistentSettings = settings; + this.metadataVersion = metadataVersion; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + } + + public RemotePersistentSettingsMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + this.blobName = blobName; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("global-metadata"), SETTING_METADATA); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + assert backingStore instanceof RemoteObjectBlobStore; + RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; + // setting the full blob path with name for future access + this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public Settings get() { + return persistentSettings; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + @Override + public RemoteObjectStore getBackingStore() { + return backingStore; + } + + @Override + public InputStream serialize() throws IOException { + return SETTINGS_METADATA_FORMAT.serialize(persistentSettings, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public Settings deserialize(InputStream inputStream) throws IOException { + return SETTINGS_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(SETTING_METADATA, blobName); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java deleted file mode 100644 index 72714bc8baba5..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteStoreFactory.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote; - -import org.opensearch.repositories.blobstore.BlobStoreRepository; - -public class RemoteStoreFactory { - - public BlobStoreRepository getBlobStoreRepository() { - return null; - } - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java new file mode 100644 index 0000000000000..3423dee0b7ce5 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.common.io.Streams; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class RemoteTemplatesMetadata extends AbstractRemoteBlobStoreObject { + + public static final String TEMPLATES_METADATA = "templates"; + + public static final ChecksumBlobStoreFormat TEMPLATES_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "templates", + METADATA_NAME_FORMAT, + TemplatesMetadata::fromXContent + ); + private TemplatesMetadata templatesMetadata; + private long metadataVersion; + private final RemoteObjectStore backingStore; + private String blobName; + private final NamedXContentRegistry xContentRegistry; + private final String clusterUUID; + public RemoteTemplatesMetadata(TemplatesMetadata templatesMetadata, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, + NamedXContentRegistry xContentRegistry) { + this.templatesMetadata = templatesMetadata; + this.metadataVersion = metadataVersion; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + } + + public RemoteTemplatesMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + this.blobName = blobName; + this.backingStore = backingStore; + this.xContentRegistry = xContentRegistry; + this.clusterUUID = clusterUUID; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("global-metadata"), TEMPLATES_METADATA); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + assert backingStore instanceof RemoteObjectBlobStore; + RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; + // setting the full blob path with name for future access + this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public TemplatesMetadata get() { + return templatesMetadata; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + @Override + public RemoteObjectStore getBackingStore() { + return backingStore; + } + + @Override + public InputStream serialize() throws IOException { + return TEMPLATES_METADATA_FORMAT.serialize(templatesMetadata, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public TemplatesMetadata deserialize(InputStream inputStream) throws IOException { + return TEMPLATES_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(TEMPLATES_METADATA, blobName); + } +} diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index 256cf8fbb4857..ffdc4729d7f21 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -45,6 +45,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING; import static org.opensearch.index.remote.RemoteIndexPath.COMBINED_PATH; import static org.opensearch.index.remote.RemoteIndexPath.SEGMENT_PATH; import static org.opensearch.index.remote.RemoteIndexPath.TRANSLOG_PATH; @@ -97,6 +98,8 @@ public RemoteIndexPathUploader( // If the remote data attributes are not present, then there is no effect of translog and segment being same or different or null. isTranslogSegmentRepoSame = isTranslogSegmentRepoSame(); Objects.requireNonNull(clusterSettings); + indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); + clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); } @Override diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index bec2d78d9af62..0babe8800fd16 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.IndexInput; import org.opensearch.action.ActionRunnable; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; @@ -23,7 +24,11 @@ import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeFileInputStream; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.index.store.exception.ChecksumCombinationException; import org.opensearch.index.translog.ChannelFactory; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import org.opensearch.threadpool.ThreadPool; @@ -38,6 +43,7 @@ import java.util.Set; import static org.opensearch.common.blobstore.BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC; +import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; /** * Service that handles remote transfer of translog and checkpoint files @@ -104,6 +110,48 @@ public void uploadBlobs( } + @Override + public void uploadBlob(InputStream inputStream, Iterable remotePath, String blobName, WritePriority writePriority, ActionListener listener) throws IOException { + assert remotePath instanceof BlobPath; + BlobPath blobPath = (BlobPath) remotePath; + final BlobContainer blobContainer = blobStore.blobContainer(blobPath); + if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { + blobContainer.writeBlob(blobName, inputStream, inputStream.available(), false); + listener.onResponse(null); + return; + } + final String resourceDescription = "BlobStoreTransferService.uploadBlob(blob=\"" + blobName + "\")"; + byte[] bytes = inputStream.readAllBytes(); + try (IndexInput input = new ByteArrayIndexInput(resourceDescription, bytes)) { + long expectedChecksum; + try { + expectedChecksum = checksumOfChecksum(input.clone(), 8); + } catch (Exception e) { + throw new ChecksumCombinationException( + "Potentially corrupted file: Checksum combination failed while combining stored checksum " + + "and calculated checksum of stored checksum", + resourceDescription, + e + ); + } + + try ( + RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + blobName, + blobName, + bytes.length, + true, + writePriority, + (size, position) -> new OffsetRangeIndexInputStream(input, size, position), + expectedChecksum, + ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported() + ) + ) { + ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), listener); + } + } + } + private void uploadBlob( TransferFileSnapshot fileSnapshot, ActionListener listener, diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java index 0894ebf500ebd..9bed1cc822fda 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java @@ -14,6 +14,7 @@ import org.opensearch.common.blobstore.FetchBlobResult; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import java.io.IOException; @@ -65,6 +66,7 @@ void uploadBlobs( * @throws IOException the exception while transferring the data */ void uploadBlob(final TransferFileSnapshot fileSnapshot, Iterable remotePath, WritePriority writePriority) throws IOException; + void uploadBlob(InputStream inputStream, Iterable remotePath, String blobName, WritePriority writePriority, ActionListener listener) throws IOException; void deleteBlobs(Iterable path, List fileNames) throws IOException; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java index 00c8cf3c0df5d..b76f8c3dbf9da 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java @@ -10,6 +10,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; @@ -29,7 +30,8 @@ public class RemoteGlobalMetadataManagerTests extends OpenSearchTestCase { public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); - remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings, threadPool); + BlobStoreTransferService blobStoreTransferService = mock(BlobStoreTransferService.class); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings, threadPool, blobStoreTransferService, "test-cluster"); } @After diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java index 78e7b78513df3..5ac0b40a9b217 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java @@ -10,6 +10,8 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; @@ -22,12 +24,14 @@ public class RemoteIndexMetadataManagerTests extends OpenSearchTestCase { private RemoteIndexMetadataManager remoteIndexMetadataManager; private BlobStoreRepository blobStoreRepository; private ClusterSettings clusterSettings; + private BlobStoreTransferService blobStoreTransferService; @Before public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, new TestThreadPool("test")); + blobStoreTransferService = mock(BlobStoreTransferService.class); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, new TestThreadPool("test"),"cluster-name", NamedXContentRegistry.EMPTY, blobStoreTransferService); } @After diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java index fb7f0612d824c..425f9fea2195e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; @@ -84,7 +85,7 @@ public void testFileNames() { .numberOfReplicas(0) .build(); - String indexMetadataFileName = RemoteIndexMetadataManager.indexMetadataFileName(indexMetadata); + String indexMetadataFileName = new RemoteIndexMetadata(indexMetadata, "cluster-uuid", null, NamedXContentRegistry.EMPTY).generateBlobFileName(); String[] splittedIndexMetadataFileName = indexMetadataFileName.split(DELIMITER); assertEquals(4, indexMetadataFileName.split(DELIMITER).length); assertEquals(METADATA_FILE_PREFIX, splittedIndexMetadataFileName[0]); From 3e1f31f12cf33fd63dda638c25f0456c91036c55 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 22 May 2024 19:25:27 +0530 Subject: [PATCH 073/133] Remove RemoteObject Store interface Signed-off-by: Sooraj Sinha --- .../remote/AbstractRemoteBlobStoreObject.java | 87 ++++++++++++++- .../remote/RemoteClusterStateService.java | 2 +- .../remote/RemoteCoordinationMetadata.java | 34 ++---- .../gateway/remote/RemoteCustomMetadata.java | 41 +++---- .../remote/RemoteGlobalMetadataManager.java | 53 +++++---- .../gateway/remote/RemoteIndexMetadata.java | 36 +++--- .../remote/RemoteIndexMetadataManager.java | 22 ++-- .../gateway/remote/RemoteObject.java | 6 +- .../gateway/remote/RemoteObjectBlobStore.java | 104 ------------------ .../gateway/remote/RemoteObjectStore.java | 27 ----- .../RemotePersistentSettingsMetadata.java | 31 ++---- .../remote/RemoteTemplatesMetadata.java | 32 ++---- .../RemoteIndexMetadataManagerTests.java | 3 +- .../remote/RemoteManifestManagerTests.java | 6 +- 14 files changed, 204 insertions(+), 280 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java index e7f7466eb5a62..542a7074528a6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java @@ -10,11 +10,37 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.concurrent.ExecutorService; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.Compressor; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; -public abstract class AbstractRemoteBlobStoreObject implements RemoteObject { +public abstract class AbstractRemoteBlobStoreObject implements RemoteObject { + + private final BlobStoreTransferService transferService; + private final BlobStoreRepository blobStoreRepository; + private final String clusterName; + private final ExecutorService executorService; + + public AbstractRemoteBlobStoreObject(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + this.transferService = blobStoreTransferService; + this.blobStoreRepository = blobStoreRepository; + this.clusterName = clusterName; + this.executorService = threadPool.executor(ThreadPool.Names.GENERIC); + } public abstract BlobPathParameters getBlobPathParameters(); + public abstract String getFullBlobName(); public String getBlobFileName() { @@ -24,8 +50,65 @@ public String getBlobFileName() { String[] pathTokens = getFullBlobName().split(PATH_DELIMITER); return getFullBlobName().split(PATH_DELIMITER)[pathTokens.length - 1]; } + public abstract String generateBlobFileName(); - public abstract RemoteObjectStore getBackingStore(); + public abstract UploadedMetadata getUploadedMetadata(); + @Override + public CheckedRunnable writeAsync(ActionListener listener) { + return () -> { + assert get() != null; + InputStream inputStream = serialize(); + transferService.uploadBlob(inputStream, getBlobPathForUpload(), getBlobFileName(), WritePriority.URGENT, listener); + }; + } + + @Override + public T read() throws IOException { + assert getFullBlobName() != null; + return deserialize( + transferService.downloadBlob(getBlobPathForDownload(), getBlobFileName())); + } + + @Override + public void readAsync(ActionListener listener) { + executorService.execute(() -> { + try { + listener.onResponse(read()); + } catch (Exception e) { + listener.onFailure(e); + } + }); + } + + public BlobPath getBlobPathForUpload() { + BlobPath blobPath = blobStoreRepository.basePath().add(RemoteClusterStateUtils.encodeString(clusterName)).add("cluster-state").add(clusterUUID()); + for (String token : getBlobPathParameters().getPathTokens()) { + blobPath = blobPath.add(token); + } + return blobPath; + } + + public BlobPath getBlobPathForDownload() { + String[] pathTokens = extractBlobPathTokens(getFullBlobName()); + BlobPath blobPath = blobStoreRepository.basePath(); + for (String token : pathTokens) { + blobPath = blobPath.add(token); + } + return blobPath; + } + + protected Compressor getCompressor() { + return blobStoreRepository.getCompressor(); + } + + protected BlobStoreRepository getBlobStoreRepository() { + return this.blobStoreRepository; + } + + private static String[] extractBlobPathTokens(String blobName) { + String[] blobNameTokens = blobName.split(PATH_DELIMITER); + return Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1); + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index f79d7e1d5fa2b..c792a78d1477e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -764,7 +764,7 @@ public void start() { } String clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings).value(); remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings,threadpool, getBlobStoreTransferService(), clusterName); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings,threadpool, clusterName, blobStoreRepository.getNamedXContentRegistry(), getBlobStoreTransferService()); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings,threadpool, clusterName, getBlobStoreTransferService()); remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(blobStoreRepository, threadpool); remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); remoteClusterStateCleanupManager.start(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java index 633a74b52b85a..d3d53aab9a4b8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java @@ -10,20 +10,20 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; import java.io.IOException; import java.io.InputStream; import java.util.List; import org.opensearch.cluster.coordination.CoordinationMetadata; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.io.Streams; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; public class RemoteCoordinationMetadata extends AbstractRemoteBlobStoreObject { @@ -36,24 +36,21 @@ public class RemoteCoordinationMetadata extends AbstractRemoteBlobStoreObject backingStore; private String blobName; - private final NamedXContentRegistry xContentRegistry; private final String clusterUUID; - public RemoteCoordinationMetadata(CoordinationMetadata coordinationMetadata, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, - NamedXContentRegistry xContentRegistry) { + public RemoteCoordinationMetadata(CoordinationMetadata coordinationMetadata, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.coordinationMetadata = coordinationMetadata; this.metadataVersion = metadataVersion; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } - public RemoteCoordinationMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + public RemoteCoordinationMetadata(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.blobName = blobName; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } @@ -77,10 +74,8 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - assert backingStore instanceof RemoteObjectBlobStore; - RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; // setting the full blob path with name for future access - this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; return blobFileName; } @@ -94,19 +89,14 @@ public String clusterUUID() { return clusterUUID; } - @Override - public RemoteObjectStore getBackingStore() { - return backingStore; - } - @Override public InputStream serialize() throws IOException { - return COORDINATION_METADATA_FORMAT.serialize(coordinationMetadata, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return COORDINATION_METADATA_FORMAT.serialize(coordinationMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); } @Override public CoordinationMetadata deserialize(InputStream inputStream) throws IOException { - return COORDINATION_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + return COORDINATION_METADATA_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java index 7220f11b8de63..a98be23df571c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java @@ -18,13 +18,15 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.common.io.Streams; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; -public class RemoteCustomMetadata extends AbstractRemoteBlobStoreObject{ +public class RemoteCustomMetadata extends AbstractRemoteBlobStoreObject { public static final String CUSTOM_METADATA = "custom"; public static final String CUSTOM_DELIMITER = "--"; @@ -32,20 +34,18 @@ public class RemoteCustomMetadata extends AbstractRemoteBlobStoreObject{ public final ChecksumBlobStoreFormat customBlobStoreFormat; private Custom custom; - private String customType; + private final String customType; private long metadataVersion; - private final RemoteObjectStore backingStore; private String blobName; - private final NamedXContentRegistry xContentRegistry; private final String clusterUUID; - public RemoteCustomMetadata(Custom custom, String customType, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, - NamedXContentRegistry xContentRegistry) { + public RemoteCustomMetadata(Custom custom, String customType, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.custom = custom; this.customType = customType; this.metadataVersion = metadataVersion; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; this.customBlobStoreFormat = new ChecksumBlobStoreFormat<>( "custom", @@ -54,11 +54,12 @@ public RemoteCustomMetadata(Custom custom, String customType, long metadataVersi ); } - public RemoteCustomMetadata(String blobName, String customType, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + public RemoteCustomMetadata(String blobName, String customType, String clusterUUID, BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.blobName = blobName; this.customType = customType; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; this.customBlobStoreFormat = new ChecksumBlobStoreFormat<>( "custom", @@ -80,7 +81,8 @@ public String getFullBlobName() { @Override public String generateBlobFileName() { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ + // String blobFileName = String.join( DELIMITER, getBlobPathParameters().getFilePrefix(), @@ -88,10 +90,8 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - assert backingStore instanceof RemoteObjectBlobStore; - RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; // setting the full blob path with name for future access - this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; return blobFileName; } @@ -105,19 +105,14 @@ public String clusterUUID() { return clusterUUID; } - @Override - public RemoteObjectStore getBackingStore() { - return backingStore; - } - @Override public InputStream serialize() throws IOException { - return customBlobStoreFormat.serialize(custom, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return customBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); } @Override public Custom deserialize(InputStream inputStream) throws IOException { - return customBlobStoreFormat.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + return customBlobStoreFormat.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index ad7cedefb965e..635e1f45dc159 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -86,21 +86,17 @@ public class RemoteGlobalMetadataManager { private final ThreadPool threadPool; private volatile TimeValue globalMetadataUploadTimeout; - private final RemoteObjectBlobStore coordinationMetadataStore; - private final RemoteObjectBlobStore persistentSettingsBlobStore; - private final RemoteObjectBlobStore templatesMetadataBlobStore; - private final RemoteObjectBlobStore customMetadataBlobStore; + private final BlobStoreTransferService blobStoreTransferService; + private final String clusterName; RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, BlobStoreTransferService blobStoreTransferService, String clusterName) { this.blobStoreRepository = blobStoreRepository; this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); this.threadPool = threadPool; + this.blobStoreTransferService = blobStoreTransferService; + this.clusterName = clusterName; clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); - this.coordinationMetadataStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); - this.persistentSettingsBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); - this.templatesMetadataBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); - this.customMetadataBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); } /** @@ -120,7 +116,7 @@ CheckedRunnable getAsyncMetadataWriteAction( ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException("Upload failed", ex)) ); - return remoteBlobStoreObject.getBackingStore().writeAsync(remoteBlobStoreObject, completionListener); + return remoteBlobStoreObject.writeAsync(completionListener); } CheckedRunnable getAsyncMetadataReadAction( @@ -134,18 +130,18 @@ CheckedRunnable getAsyncMetadataReadAction( ) { AbstractRemoteBlobStoreObject remoteBlobStoreObject; if (component.equals(COORDINATION_METADATA)) { - remoteBlobStoreObject = new RemoteCoordinationMetadata(uploadFilename, clusterUUID, coordinationMetadataStore, blobStoreRepository.getNamedXContentRegistry()); + remoteBlobStoreObject = new RemoteCoordinationMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else if (component.equals(TEMPLATES_METADATA)) { - remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, templatesMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else if (component.equals(SETTING_METADATA)) { - remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, persistentSettingsBlobStore, blobStoreRepository.getNamedXContentRegistry()); + remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else if (component.equals(CUSTOM_METADATA)) { - remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, customMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else { throw new RemoteStateTransferException("Unknown component " + componentName); } ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); - return () -> remoteBlobStoreObject.getBackingStore().readAsync(remoteBlobStoreObject, actionListener); + return () -> remoteBlobStoreObject.readAsync(actionListener); // String[] splitName = uploadFilename.split("/"); // return () -> componentBlobStore.readAsync( // globalMetadataContainer(clusterName, clusterUUID), @@ -212,8 +208,8 @@ public CoordinationMetadata getCoordinationMetadata(String clusterUUID, String c // Fetch Coordination metadata if (coordinationMetadataFileName != null) { RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(coordinationMetadataFileName, clusterUUID, - coordinationMetadataStore, blobStoreRepository.getNamedXContentRegistry()); - return remoteCoordinationMetadata.getBackingStore().read(remoteCoordinationMetadata); + blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + return remoteCoordinationMetadata.read(); } else { return CoordinationMetadata.EMPTY_METADATA; } @@ -230,8 +226,8 @@ public Settings getSettingsMetadata(String clusterUUID, String settingsMetadataF // Fetch Settings metadata if (settingsMetadataFileName != null) { RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(settingsMetadataFileName, clusterUUID, - persistentSettingsBlobStore, blobStoreRepository.getNamedXContentRegistry()); - return remotePersistentSettingsMetadata.getBackingStore().read(remotePersistentSettingsMetadata); + blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + return remotePersistentSettingsMetadata.read(); } else { return Settings.EMPTY; } @@ -248,8 +244,8 @@ public TemplatesMetadata getTemplatesMetadata(String clusterUUID, String templat // Fetch Templates metadata if (templatesMetadataFileName != null) { RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(templatesMetadataFileName, clusterUUID, - templatesMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); - return remoteTemplatesMetadata.getBackingStore().read(remoteTemplatesMetadata); + blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + return remoteTemplatesMetadata.read(); } else { return TemplatesMetadata.EMPTY_METADATA; } @@ -265,8 +261,8 @@ public Metadata.Custom getCustomsMetadata(String clusterUUID, String customMetad requireNonNull(customMetadataFileName); try { // Fetch Custom metadata - RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, customMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); - return remoteCustomMetadata.getBackingStore().read(remoteCustomMetadata); + RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + return remoteCustomMetadata.read(); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), @@ -302,14 +298,17 @@ Map getUpdatedCustoms(ClusterState currentState, Cluste private AbstractRemoteBlobStoreObject getRemoteObject(Object object, long metadataVersion, String clusterUUID, String customType) { if (object instanceof CoordinationMetadata) { - return new RemoteCoordinationMetadata((CoordinationMetadata) object, metadataVersion, clusterUUID, coordinationMetadataStore, - blobStoreRepository.getNamedXContentRegistry()); + return new RemoteCoordinationMetadata((CoordinationMetadata) object, metadataVersion, clusterUUID, blobStoreTransferService, + blobStoreRepository, clusterName, threadPool); } else if (object instanceof Settings) { - return new RemotePersistentSettingsMetadata((Settings) object, metadataVersion, clusterUUID, persistentSettingsBlobStore, blobStoreRepository.getNamedXContentRegistry()); + return new RemotePersistentSettingsMetadata((Settings) object, metadataVersion, clusterUUID, blobStoreTransferService, + blobStoreRepository, clusterName, threadPool); } else if (object instanceof TemplatesMetadata) { - return new RemoteTemplatesMetadata((TemplatesMetadata) object, metadataVersion, clusterUUID, templatesMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + return new RemoteTemplatesMetadata((TemplatesMetadata) object, metadataVersion, clusterUUID, blobStoreTransferService, + blobStoreRepository, clusterName, threadPool); } else if (object instanceof Custom) { - return new RemoteCustomMetadata((Custom) object, customType ,metadataVersion, clusterUUID, customMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + return new RemoteCustomMetadata((Custom) object, customType ,metadataVersion, clusterUUID, blobStoreTransferService, + blobStoreRepository, clusterName, threadPool); } throw new RemoteStateTransferException("Remote object cannot be created for " + object.getClass()); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java index 1543dc0ed11a7..a914e6ef329ee 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java @@ -16,13 +16,16 @@ import java.util.List; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.io.Streams; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; public class RemoteIndexMetadata extends AbstractRemoteBlobStoreObject { + public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; public static final ChecksumBlobStoreFormat INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( @@ -32,23 +35,22 @@ public class RemoteIndexMetadata extends AbstractRemoteBlobStoreObject backingStore; private String blobName; - private final NamedXContentRegistry xContentRegistry; private final String clusterUUID; - public RemoteIndexMetadata(IndexMetadata indexMetadata, String clusterUUID, RemoteObjectBlobStore backingStore, - NamedXContentRegistry xContentRegistry) { + public RemoteIndexMetadata(IndexMetadata indexMetadata, String clusterUUID, + BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.indexMetadata = indexMetadata; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } - public RemoteIndexMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + public RemoteIndexMetadata(String blobName, String clusterUUID, + BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.blobName = blobName; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } @@ -82,18 +84,11 @@ public String generateBlobFileName() { String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during reads we read last // place to determine codec version. ); - assert backingStore instanceof RemoteObjectBlobStore; - RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; // setting the full blob path with name for future access - this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; return blobFileName; } - @Override - public RemoteObjectStore getBackingStore() { - return backingStore; - } - @Override public UploadedMetadata getUploadedMetadata() { assert blobName != null; @@ -102,13 +97,14 @@ public UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - return INDEX_METADATA_FORMAT.serialize(indexMetadata, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return INDEX_METADATA_FORMAT.serialize(indexMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) + .streamInput(); } @Override public IndexMetadata deserialize(InputStream inputStream) throws IOException { // Blob name parameter is redundant - return INDEX_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + return INDEX_METADATA_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index 509cffce86b2c..702a4c03e049b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -47,19 +47,19 @@ public class RemoteIndexMetadataManager { private final BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; - private final NamedXContentRegistry namedXContentRegistry; - private final RemoteObjectBlobStore remoteIndexMetadataBlobStore; + private final BlobStoreTransferService blobStoreTransferService; private volatile TimeValue indexMetadataUploadTimeout; + private final String clusterName; public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, String clusterName, - NamedXContentRegistry namedXContentRegistry, BlobStoreTransferService blobStoreTransferService) { + BlobStoreTransferService blobStoreTransferService) { this.blobStoreRepository = blobStoreRepository; this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); this.threadPool = threadPool; clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); - this.remoteIndexMetadataBlobStore = new RemoteObjectBlobStore<>(threadPool.executor(ThreadPool.Names.GENERIC), blobStoreTransferService, blobStoreRepository, clusterName); - this.namedXContentRegistry = namedXContentRegistry; + this.blobStoreTransferService = blobStoreTransferService; + this.clusterName = clusterName; } /** @@ -70,14 +70,14 @@ public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, Clust */ CheckedRunnable getIndexMetadataAsyncAction(IndexMetadata indexMetadata, String clusterUUID, LatchedActionListener latchedActionListener) { - RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(indexMetadata, clusterUUID, remoteIndexMetadataBlobStore, namedXContentRegistry); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( remoteIndexMetadata.getUploadedMetadata() ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) ); - return remoteIndexMetadataBlobStore.writeAsync(remoteIndexMetadata, completionListener); + return remoteIndexMetadata.writeAsync(completionListener); } CheckedRunnable getAsyncIndexMetadataReadAction( @@ -85,12 +85,12 @@ CheckedRunnable getAsyncIndexMetadataReadAction( String uploadedFilename, LatchedActionListener latchedActionListener ) { - RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, remoteIndexMetadataBlobStore, blobStoreRepository.getNamedXContentRegistry()); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); ActionListener actionListener = ActionListener.wrap( //todo change dummy response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, "dummy")), latchedActionListener::onFailure); - return () -> remoteIndexMetadataBlobStore.readAsync(remoteIndexMetadata, actionListener); + return () -> remoteIndexMetadata.readAsync(actionListener); // String[] splitPath = uploadedFilename.split("/"); // return () -> INDEX_METADATA_FORMAT.readAsync( // indexMetadataContainer(clusterName, clusterUUID, splitPath[0]), @@ -113,9 +113,9 @@ IndexMetadata getIndexMetadata( ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata, String clusterUUID, int manifestCodecVersion ) { RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(RemoteClusterStateUtils.getFormattedFileName( - uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, remoteIndexMetadataBlobStore, namedXContentRegistry); + uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); try { - return remoteIndexMetadataBlobStore.read(remoteIndexMetadata); + return remoteIndexMetadata.read(); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading IndexMetadata - %s", uploadedIndexMetadata.getUploadedFilename()), diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java index e08d76ad3a6b5..8802ab0e96c5a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java @@ -11,13 +11,17 @@ import java.io.IOException; import java.io.InputStream; import org.opensearch.common.CheckedRunnable; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; public interface RemoteObject { public T get(); public String clusterUUID(); - public RemoteObjectStore getBackingStore(); public InputStream serialize() throws IOException; public T deserialize(InputStream inputStream) throws IOException; + public CheckedRunnable writeAsync(ActionListener listener); + public T read() throws IOException; + public void readAsync(ActionListener listener); + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java deleted file mode 100644 index a83d1178c7b20..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectBlobStore.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.concurrent.ExecutorService; -import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.blobstore.stream.write.WritePriority; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.compress.Compressor; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; - -public class RemoteObjectBlobStore implements RemoteObjectStore { - - private static final String PATH_DELIMITER = "/"; - - private final ExecutorService executorService; - private final BlobStoreTransferService transferService; - private final BlobStoreRepository blobStoreRepository; - private final String clusterName; - - public RemoteObjectBlobStore(ExecutorService executorService, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository remoteStoreFactory, String clusterName) { - this.executorService = executorService; - this.transferService = blobStoreTransferService; - this.blobStoreRepository = remoteStoreFactory; - this.clusterName = clusterName; - } - - @Override - public CheckedRunnable writeAsync(RemoteObject remoteObject, ActionListener listener) { - return () -> { - AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; - InputStream inputStream = abstractRemoteBlobStoreObject.serialize(); - transferService.uploadBlob(inputStream, getBlobPathForUpload(remoteObject), abstractRemoteBlobStoreObject.getBlobFileName(), WritePriority.URGENT, - listener); - }; - } - - @Override - public T read(RemoteObject remoteObject) throws IOException { - AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; - return remoteObject.deserialize( - transferService.downloadBlob(getBlobPathForDownload(remoteObject), abstractRemoteBlobStoreObject.getBlobFileName())); - } - - @Override - public void readAsync(RemoteObject remoteObject, ActionListener listener) { - executorService.execute(() -> { - try { - listener.onResponse(read(remoteObject)); - } catch (Exception e) { - listener.onFailure(e); - } - }); - } - - @Override - public Compressor getCompressor() { - return blobStoreRepository.getCompressor(); - } - - public BlobPath getBlobPathForUpload(RemoteObject remoteObject) { - BlobPath blobPath = blobStoreRepository.basePath().add(RemoteClusterStateUtils.encodeString(clusterName)).add("cluster-state").add(remoteObject.clusterUUID()); - AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; - for (String token : abstractRemoteBlobStoreObject.getBlobPathParameters().getPathTokens()) { - blobPath = blobPath.add(token); - } - return blobPath; - } - - public BlobPath getBlobPathForDownload(RemoteObject remoteObject) { - AbstractRemoteBlobStoreObject abstractRemoteBlobStoreObject = (AbstractRemoteBlobStoreObject) remoteObject; - String[] pathTokens = extractBlobPathTokens(abstractRemoteBlobStoreObject.getFullBlobName()); - BlobPath blobPath = blobStoreRepository.basePath(); - for (String token : pathTokens) { - blobPath = blobPath.add(token); - } - return blobPath; - } - - private static String extractBlobFileName(String blobName) { - // todo add handling for *.dat extension - assert blobName != null; - String[] blobNameTokens = blobName.split(PATH_DELIMITER); - return blobNameTokens[blobNameTokens.length - 1]; - } - - private static String[] extractBlobPathTokens(String blobName) { - String[] blobNameTokens = blobName.split(PATH_DELIMITER); - return Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1); - } - - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java deleted file mode 100644 index 54ac83d5d83c8..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObjectStore.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote; - -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.CompletableFuture; -import java.util.function.Function; -import org.opensearch.common.CheckedRunnable; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.compress.Compressor; -import org.opensearch.core.xcontent.ToXContent; - -public interface RemoteObjectStore { - - public CheckedRunnable writeAsync(RemoteObject remoteObject, ActionListener listener); - public T read(RemoteObject remoteObject) throws IOException; - public void readAsync(RemoteObject remoteObject, ActionListener listener); - public Compressor getCompressor(); - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java index 3a9e515a03ad0..f23cd6dcc1eb8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java @@ -17,11 +17,13 @@ import java.util.List; import org.opensearch.common.io.Streams; import org.opensearch.common.settings.Settings; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobStoreObject { @@ -35,24 +37,21 @@ public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobStoreObj private Settings persistentSettings; private long metadataVersion; - private final RemoteObjectStore backingStore; private String blobName; - private final NamedXContentRegistry xContentRegistry; private final String clusterUUID; - public RemotePersistentSettingsMetadata(Settings settings, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, - NamedXContentRegistry xContentRegistry) { + public RemotePersistentSettingsMetadata(Settings settings, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.persistentSettings = settings; this.metadataVersion = metadataVersion; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } - public RemotePersistentSettingsMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + public RemotePersistentSettingsMetadata(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.blobName = blobName; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } @@ -75,10 +74,8 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - assert backingStore instanceof RemoteObjectBlobStore; - RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; // setting the full blob path with name for future access - this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; return blobFileName; } @@ -92,19 +89,15 @@ public String clusterUUID() { return clusterUUID; } - @Override - public RemoteObjectStore getBackingStore() { - return backingStore; - } @Override public InputStream serialize() throws IOException { - return SETTINGS_METADATA_FORMAT.serialize(persistentSettings, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return SETTINGS_METADATA_FORMAT.serialize(persistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); } @Override public Settings deserialize(InputStream inputStream) throws IOException { - return SETTINGS_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + return SETTINGS_METADATA_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java index 3423dee0b7ce5..9c302993536a9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java @@ -17,11 +17,13 @@ import java.util.List; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.common.io.Streams; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; public class RemoteTemplatesMetadata extends AbstractRemoteBlobStoreObject { @@ -34,23 +36,20 @@ public class RemoteTemplatesMetadata extends AbstractRemoteBlobStoreObject backingStore; private String blobName; - private final NamedXContentRegistry xContentRegistry; private final String clusterUUID; - public RemoteTemplatesMetadata(TemplatesMetadata templatesMetadata, long metadataVersion, String clusterUUID, RemoteObjectBlobStore backingStore, - NamedXContentRegistry xContentRegistry) { + public RemoteTemplatesMetadata(TemplatesMetadata templatesMetadata, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.templatesMetadata = templatesMetadata; this.metadataVersion = metadataVersion; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } - public RemoteTemplatesMetadata(String blobName, String clusterUUID, RemoteObjectStore backingStore, NamedXContentRegistry xContentRegistry) { + public RemoteTemplatesMetadata(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.blobName = blobName; - this.backingStore = backingStore; - this.xContentRegistry = xContentRegistry; this.clusterUUID = clusterUUID; } @@ -74,10 +73,8 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - assert backingStore instanceof RemoteObjectBlobStore; - RemoteObjectBlobStore blobStore = (RemoteObjectBlobStore) backingStore; // setting the full blob path with name for future access - this.blobName = blobStore.getBlobPathForUpload(this).buildAsString() + blobFileName; + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; return blobFileName; } @@ -91,19 +88,14 @@ public String clusterUUID() { return clusterUUID; } - @Override - public RemoteObjectStore getBackingStore() { - return backingStore; - } - @Override public InputStream serialize() throws IOException { - return TEMPLATES_METADATA_FORMAT.serialize(templatesMetadata, generateBlobFileName(), getBackingStore().getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return TEMPLATES_METADATA_FORMAT.serialize(templatesMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); } @Override public TemplatesMetadata deserialize(InputStream inputStream) throws IOException { - return TEMPLATES_METADATA_FORMAT.deserialize(blobName, xContentRegistry, Streams.readFully(inputStream)); + return TEMPLATES_METADATA_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); } @Override diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java index 5ac0b40a9b217..87c19e883fa2f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java @@ -10,7 +10,6 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; @@ -31,7 +30,7 @@ public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); blobStoreTransferService = mock(BlobStoreTransferService.class); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, new TestThreadPool("test"),"cluster-name", NamedXContentRegistry.EMPTY, blobStoreTransferService); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, new TestThreadPool("test"),"cluster-name", blobStoreTransferService); } @After diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java index 425f9fea2195e..10a718f6a627e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java @@ -19,12 +19,14 @@ import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.junit.After; import org.junit.Before; import java.io.IOException; +import org.opensearch.threadpool.TestThreadPool; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; @@ -43,12 +45,14 @@ public class RemoteManifestManagerTests extends OpenSearchTestCase { private ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; private BlobStore blobStore; + private BlobStoreTransferService blobStoreTransferService; @Before public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, "test-node-id"); + blobStoreTransferService = mock(BlobStoreTransferService.class); blobStore = mock(BlobStore.class); when(blobStoreRepository.blobStore()).thenReturn(blobStore); } @@ -85,7 +89,7 @@ public void testFileNames() { .numberOfReplicas(0) .build(); - String indexMetadataFileName = new RemoteIndexMetadata(indexMetadata, "cluster-uuid", null, NamedXContentRegistry.EMPTY).generateBlobFileName(); + String indexMetadataFileName = new RemoteIndexMetadata(indexMetadata, "cluster-uuid", blobStoreTransferService, blobStoreRepository, "cluster-name", new TestThreadPool("test")).generateBlobFileName(); String[] splittedIndexMetadataFileName = indexMetadataFileName.split(DELIMITER); assertEquals(4, indexMetadataFileName.split(DELIMITER).length); assertEquals(METADATA_FILE_PREFIX, splittedIndexMetadataFileName[0]); From 043a170bd37a4ec9985387a22f6ece2ba83e0ea0 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 23 May 2024 08:48:12 +0530 Subject: [PATCH 074/133] Reverting dev testing configuration --- .../opensearch/gradle/testclusters/OpenSearchNode.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java index 25df09ed9e4ac..268de50340cbf 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java @@ -1158,22 +1158,15 @@ private void createConfiguration() { if (nodeName != null) { baseConfig.put("node.name", nodeName); } - baseConfig.put("path.repo", "/Users/soosinha/os_data/repos"); + baseConfig.put("path.repo", confPathRepo.toAbsolutePath().toString()); baseConfig.put("path.data", confPathData.toAbsolutePath().toString()); baseConfig.put("path.logs", confPathLogs.toAbsolutePath().toString()); - baseConfig.put("cluster.remote_store.state.enabled", "true"); - baseConfig.put("path.shared_data", workingDir.resolve("sharedData").toString()); baseConfig.put("node.attr.testattr", "test"); if (StringUtils.isNotBlank(zone)) { baseConfig.put("cluster.routing.allocation.awareness.attributes", "zone"); baseConfig.put("node.attr.zone", zone); } - baseConfig.put("node.attr.remote_store.state.repository", "my-fs-repository"); - baseConfig.put("node.attr.remote_store.segment.repository", "my-fs-repository"); - baseConfig.put("node.attr.remote_store.translog.repository", "my-fs-repository"); - baseConfig.put("node.attr.remote_store.repository.my-fs-repository.type", "fs"); - baseConfig.put("node.attr.remote_store.repository.my-fs-repository.settings.location", "/Users/soosinha/os_data/repos/gradlerepo"); baseConfig.put("node.portsfile", "true"); baseConfig.put("http.port", httpPort); baseConfig.put("transport.port", transportPort); From 3e3dfc87b11ae3e83c1944389f505e76076e8b3b Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 23 May 2024 12:56:53 +0530 Subject: [PATCH 075/133] Refactor discoverynodes and clusterblocks --- .../gateway/remote/RemoteClusterBlocks.java | 109 ++++++++++++++++++ .../RemoteClusterStateAttributesManager.java | 98 +++++----------- .../RemoteClusterStateCleanupManager.java | 2 +- .../remote/RemoteClusterStateService.java | 22 +--- .../gateway/remote/RemoteDiscoveryNodes.java | 107 +++++++++++++++++ .../remote/RemoteGlobalMetadataManager.java | 9 -- .../gateway/remote/RemoteIndexMetadata.java | 3 +- .../remote/RemoteIndexMetadataManager.java | 24 +--- 8 files changed, 253 insertions(+), 121 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterBlocks.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteDiscoveryNodes.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterBlocks.java new file mode 100644 index 0000000000000..8239ca76d3b6b --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterBlocks.java @@ -0,0 +1,109 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.common.io.Streams; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteClusterBlocks extends AbstractRemoteBlobStoreObject { + + public static final String CLUSTER_BLOCKS = "blocks"; + public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( + "blocks", + METADATA_NAME_FORMAT, + ClusterBlocks::fromXContent + ); + + private ClusterBlocks clusterBlocks; + private long stateVersion; + private String blobName; + private final String clusterUUID; + + public RemoteClusterBlocks(ClusterBlocks clusterBlocks, long stateVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.clusterBlocks = clusterBlocks; + this.stateVersion = stateVersion; + this.clusterUUID = clusterUUID; + } + + public RemoteClusterBlocks(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, + String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.blobName = blobName; + this.clusterUUID = clusterUUID; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("transient"), CLUSTER_BLOCKS); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/transient/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + // setting the full blob path with name for future access + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(CLUSTER_BLOCKS, blobName); + } + + @Override + public ClusterBlocks get() { + return clusterBlocks; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + @Override + public InputStream serialize() throws IOException { + return CLUSTER_BLOCKS_FORMAT.serialize(clusterBlocks, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public ClusterBlocks deserialize(InputStream inputStream) throws IOException { + return CLUSTER_BLOCKS_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 6dee7e057e123..ae1f1ecc36cde 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -18,7 +18,9 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; @@ -35,23 +37,17 @@ public class RemoteClusterStateAttributesManager { public static final String DISCOVERY_NODES = "nodes"; public static final String CLUSTER_BLOCKS = "blocks"; public static final String CUSTOM_PREFIX = "custom"; - public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( - "nodes", - METADATA_NAME_FORMAT, - DiscoveryNodes::fromXContent - ); - public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( - "blocks", - METADATA_NAME_FORMAT, - ClusterBlocks::fromXContent - ); public static final int CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION = 1; + private final BlobStoreTransferService blobStoreTransferService; private final BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; + private final String clusterName; - RemoteClusterStateAttributesManager(BlobStoreRepository repository, ThreadPool threadPool) { + RemoteClusterStateAttributesManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository repository, ThreadPool threadPool, String clusterName) { + this.blobStoreTransferService = blobStoreTransferService; this.blobStoreRepository = repository; this.threadPool = threadPool; + this.clusterName = clusterName; } /** @@ -60,82 +56,48 @@ public class RemoteClusterStateAttributesManager { CheckedRunnable getAsyncMetadataWriteAction( ClusterState clusterState, String component, - ChecksumBlobStoreFormat componentMetadataBlobStore, ToXContent componentData, LatchedActionListener latchedActionListener ) { - final BlobContainer remoteStateAttributeContainer = clusterStateAttributeContainer(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()); - final String componentMetadataFilename = clusterStateAttributeFileName(component, clusterState.metadata().version()); + AbstractRemoteBlobStoreObject remoteObject = getRemoteObject(componentData, clusterState.version(), clusterState.metadata().clusterUUID()); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( - new ClusterMetadataManifest.UploadedMetadataAttribute(component, componentMetadataFilename) + remoteObject.getUploadedMetadata() ), ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) ); - return () -> componentMetadataBlobStore.writeAsyncWithUrgentPriority( - componentData, - remoteStateAttributeContainer, - componentMetadataFilename, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); + return remoteObject.writeAsync(completionListener); + } + + private AbstractRemoteBlobStoreObject getRemoteObject(ToXContent componentData, long stateVersion, String clusterUUID) { + if (componentData instanceof DiscoveryNodes) { + return new RemoteDiscoveryNodes((DiscoveryNodes)componentData, stateVersion, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } else if (componentData instanceof ClusterBlocks) { + return new RemoteClusterBlocks((ClusterBlocks) componentData, stateVersion, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } else { + throw new RemoteStateTransferException("Remote object not found for "+ componentData.getClass()); + } } public CheckedRunnable getAsyncMetadataReadAction( - String clusterName, String clusterUUID, String component, String uploadedFilename, - ChecksumBlobStoreFormat componentBlobStore, LatchedActionListener listener ) { - String[] splitFilename = uploadedFilename.split("/"); - return () -> componentBlobStore.readAsync( - clusterStateAttributeContainer(clusterName, clusterUUID), - splitFilename[splitFilename.length -1], - blobStoreRepository.getNamedXContentRegistry(), - threadPool.executor(ThreadPool.Names.GENERIC), - ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure) - ); + AbstractRemoteBlobStoreObject remoteObject = getRemoteObject(component, uploadedFilename, clusterUUID); + ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); + return () -> remoteObject.readAsync(actionListener); } - public ToXContent readMetadata(ChecksumBlobStoreFormat componentMetadataBlobStore, String clusterName, String clusterUUID, String fileName) { - final BlobContainer remoteStateAttributeContainer = clusterStateAttributeContainer(clusterName, clusterUUID); - try { - // Fetch custom metadata - if (fileName != null) { - String[] splitPath = fileName.split("/"); - return componentMetadataBlobStore.read( - remoteStateAttributeContainer, - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); - } else { - return TemplatesMetadata.EMPTY_METADATA; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", fileName), - e - ); + private AbstractRemoteBlobStoreObject getRemoteObject(String component, String blobName, String clusterUUID) { + if (component.equals(RemoteDiscoveryNodes.DISCOVERY_NODES)) { + return new RemoteDiscoveryNodes(blobName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } else if (component.equals(RemoteClusterBlocks.CLUSTER_BLOCKS)) { + return new RemoteClusterBlocks(blobName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } else { + throw new RemoteStateTransferException("Remote object not found for "+ component); } } - private BlobContainer clusterStateAttributeContainer(String clusterName, String clusterUUID) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ - return blobStoreRepository.blobStore() - .blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID)); - } - - private static String clusterStateAttributeFileName(String componentPrefix, Long metadataVersion) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/______ - return String.join( - DELIMITER, - componentPrefix, - RemoteStoreUtils.invertLong(metadataVersion), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) - ); - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 0d6c9b715a331..090dae144baf6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -37,7 +37,7 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c792a78d1477e..78263991014de 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -10,24 +10,18 @@ import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteReadResult; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_DELIMITER; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; @@ -79,7 +73,6 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -88,7 +81,6 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; /** @@ -525,7 +517,6 @@ private UploadedMetadataResults writeMetadataInParallel( remoteClusterStateAttributesManager.getAsyncMetadataWriteAction( clusterState, DISCOVERY_NODES, - DISCOVERY_NODES_FORMAT, clusterState.nodes(), listener ) @@ -537,7 +528,6 @@ private UploadedMetadataResults writeMetadataInParallel( remoteClusterStateAttributesManager.getAsyncMetadataWriteAction( clusterState, CLUSTER_BLOCKS, - CLUSTER_BLOCKS_FORMAT, clusterState.blocks(), listener ) @@ -765,7 +755,7 @@ public void start() { String clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings).value(); remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings,threadpool, getBlobStoreTransferService(), clusterName); remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings,threadpool, clusterName, getBlobStoreTransferService()); - remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(blobStoreRepository, threadpool); + remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(getBlobStoreTransferService(), blobStoreRepository, threadpool, clusterName); remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); remoteClusterStateCleanupManager.start(); } @@ -894,7 +884,6 @@ private ClusterState readClusterStateInParallel( CUSTOM_METADATA, entry.getKey(), entry.getValue().getUploadedFilename(), - new ChecksumBlobStoreFormat("custom", METADATA_NAME_FORMAT, (parser -> Metadata.Custom.fromXContent((XContentParser) parser, entry.getKey()))), listener ) ); @@ -908,7 +897,6 @@ private ClusterState readClusterStateInParallel( COORDINATION_METADATA, COORDINATION_METADATA, manifest.getCoordinationMetadata().getUploadedFilename(), - COORDINATION_METADATA_FORMAT, listener ) ); @@ -922,7 +910,6 @@ private ClusterState readClusterStateInParallel( SETTING_METADATA, SETTING_METADATA, manifest.getSettingsMetadata().getUploadedFilename(), - SETTINGS_METADATA_FORMAT, listener ) ); @@ -936,7 +923,6 @@ private ClusterState readClusterStateInParallel( TEMPLATES_METADATA, TEMPLATES_METADATA, manifest.getTemplatesMetadata().getUploadedFilename(), - TEMPLATES_METADATA_FORMAT, listener ) ); @@ -945,11 +931,9 @@ private ClusterState readClusterStateInParallel( if (readDiscoveryNodes) { asyncMetadataReadActions.add( remoteClusterStateAttributesManager.getAsyncMetadataReadAction( - clusterName, clusterUUID, DISCOVERY_NODES, manifest.getDiscoveryNodesMetadata().getUploadedFilename(), - DISCOVERY_NODES_FORMAT, listener ) ); @@ -958,11 +942,9 @@ private ClusterState readClusterStateInParallel( if (readClusterBlocks) { asyncMetadataReadActions.add( remoteClusterStateAttributesManager.getAsyncMetadataReadAction( - clusterName, clusterUUID, CLUSTER_BLOCKS, manifest.getClusterBlocksMetadata().getUploadedFilename(), - CLUSTER_BLOCKS_FORMAT, listener ) ); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteDiscoveryNodes.java new file mode 100644 index 0000000000000..cfa74f683ac4f --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteDiscoveryNodes.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.io.Streams; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteDiscoveryNodes extends AbstractRemoteBlobStoreObject { + + public static final String DISCOVERY_NODES = "nodes"; + public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( + "nodes", + METADATA_NAME_FORMAT, + DiscoveryNodes::fromXContent + ); + + private DiscoveryNodes discoveryNodes; + private long stateVersion; + private String blobName; + private final String clusterUUID; + + public RemoteDiscoveryNodes(DiscoveryNodes discoveryNodes, long stateVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.discoveryNodes = discoveryNodes; + this.stateVersion = stateVersion; + this.clusterUUID = clusterUUID; + } + + public RemoteDiscoveryNodes(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.blobName = blobName; + this.clusterUUID = clusterUUID; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("transient"), DISCOVERY_NODES); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/transient/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + // setting the full blob path with name for future access + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(DISCOVERY_NODES, blobName); + } + + @Override + public DiscoveryNodes get() { + return discoveryNodes; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + @Override + public InputStream serialize() throws IOException { + return DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public DiscoveryNodes deserialize(InputStream inputStream) throws IOException { + return DISCOVERY_NODES_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 635e1f45dc159..5ad7c2de6b5e3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -125,7 +125,6 @@ CheckedRunnable getAsyncMetadataReadAction( String component, String componentName, String uploadFilename, - ChecksumBlobStoreFormat componentBlobStore, LatchedActionListener listener ) { AbstractRemoteBlobStoreObject remoteBlobStoreObject; @@ -142,14 +141,6 @@ CheckedRunnable getAsyncMetadataReadAction( } ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); return () -> remoteBlobStoreObject.readAsync(actionListener); -// String[] splitName = uploadFilename.split("/"); -// return () -> componentBlobStore.readAsync( -// globalMetadataContainer(clusterName, clusterUUID), -// splitName[splitName.length - 1], -// blobStoreRepository.getNamedXContentRegistry(), -// threadPool.executor(ThreadPool.Names.GENERIC), -// ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure) -// ); } Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java index a914e6ef329ee..6f031c3e020d3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java @@ -33,6 +33,7 @@ public class RemoteIndexMetadata extends AbstractRemoteBlobStoreObject INDEX_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "index-metadata", - METADATA_NAME_FORMAT, - IndexMetadata::fromXContent - ); - public static final String INDEX_PATH_TOKEN = "index"; - private final BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; @@ -87,20 +78,9 @@ CheckedRunnable getAsyncIndexMetadataReadAction( ) { RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); ActionListener actionListener = ActionListener.wrap( - //todo change dummy - response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, "dummy")), + response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, response.getIndexName())), latchedActionListener::onFailure); return () -> remoteIndexMetadata.readAsync(actionListener); -// String[] splitPath = uploadedFilename.split("/"); -// return () -> INDEX_METADATA_FORMAT.readAsync( -// indexMetadataContainer(clusterName, clusterUUID, splitPath[0]), -// splitPath[splitPath.length - 1], -// blobStoreRepository.getNamedXContentRegistry(), -// threadPool.executor(ThreadPool.Names.GENERIC), -// ActionListener.wrap( -// response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, "dummy")), -// latchedActionListener::onFailure) -// ); } /** From 89a10ace1e2a97de922eee39074cd90ff933d72a Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 23 May 2024 13:10:06 +0530 Subject: [PATCH 076/133] Async write flow for routing table --- .../remote/RemoteRoutingTableService.java | 187 ++++++++++++------ .../remote/ClusterMetadataManifest.java | 33 +++- .../remote/RemoteClusterStateService.java | 69 ++++--- .../remote/RemoteClusterStateUtils.java | 6 +- .../index/remote/RemoteStoreEnums.java | 30 +-- .../index/remote/RemoteStorePathStrategy.java | 9 +- 6 files changed, 223 insertions(+), 111 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 858583d2ddcdd..499296a15e6b6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,41 +10,39 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.Version; +import org.apache.lucene.store.IndexInput; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; -import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.index.Index; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest; -import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStream; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStreamReader; +import org.opensearch.index.remote.RemoteStoreEnums; +import org.opensearch.index.remote.RemoteStorePathStrategy; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; -import java.io.*; import java.io.Closeable; import java.io.IOException; @@ -53,11 +51,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; @@ -78,100 +78,157 @@ public class RemoteRoutingTableService implements Closeable { Setting.Property.Final ); public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing"; + public static final String INDEX_ROUTING_FILE_PREFIX = "index_routing"; public static final String DELIMITER = "__"; - + public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--"; private static final Logger logger = LogManager.getLogger(RemoteRoutingTableService.class); private final Settings settings; private final Supplier repositoriesService; - private final ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; public RemoteRoutingTableService(Supplier repositoriesService, Settings settings, - ClusterSettings clusterSettings, ThreadPool threadPool) { + ThreadPool threadPool) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; - this.clusterSettings = clusterSettings; this.threadPool = threadPool; } - public List writeFullRoutingTable(ClusterState clusterState, String previousClusterUUID) { + public List getChangedIndicesRouting( ClusterState previousClusterState, + ClusterState clusterState) { + Map previousIndexRoutingTable = previousClusterState.getRoutingTable().getIndicesRouting(); + List changedIndicesRouting = new ArrayList<>(); + for (IndexRoutingTable indexRouting : clusterState.getRoutingTable().getIndicesRouting().values()) { + if (!(previousIndexRoutingTable.containsKey(indexRouting.getIndex().getName()) && indexRouting.equals(previousIndexRoutingTable.get(indexRouting.getIndex().getName())))) { + changedIndicesRouting.add(indexRouting); + logger.info("changedIndicesRouting {}", indexRouting.prettyPrint()); + } + } + return changedIndicesRouting; + } + + public CheckedRunnable getIndexRoutingAsyncAction( + ClusterState clusterState, + IndexRoutingTable indexRouting, + LatchedActionListener latchedActionListener + ) throws IOException { - //batch index count and parallelize - RoutingTable currentRoutingTable = clusterState.getRoutingTable(); - List uploadedIndices = new ArrayList<>(); BlobPath custerMetadataBasePath = getCusterMetadataBasePath(blobStoreRepository, clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID()); - for (IndexRoutingTable indexRouting : currentRoutingTable.getIndicesRouting().values()) { - uploadedIndices.add(uploadIndex(indexRouting, custerMetadataBasePath)); + clusterState.metadata().clusterUUID()).add(INDEX_ROUTING_PATH_TOKEN); + logger.info("custerMetadataBasePath {}", custerMetadataBasePath); + + BlobPath path = RemoteStoreEnums.PathType.HASHED_PREFIX.path(RemoteStorePathStrategy.PathInput.builder() + .basePath(custerMetadataBasePath) + .indexUUID(indexRouting.getIndex().getUUID()) + .build(), + RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64); + logger.info("path from prefix hasd {}", path); + final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(path); + + final String fileName = getIndexRoutingFileName(); + logger.info("fileName {}", fileName); + + ActionListener completionListener = ActionListener.wrap( + resp -> latchedActionListener.onResponse( + new ClusterMetadataManifest.UploadedIndexMetadata( + + indexRouting.getIndex().getName(), + indexRouting.getIndex().getUUID(), + path.buildAsString() + fileName, + INDEX_ROUTING_METADATA_PREFIX + ) + ), + ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(indexRouting.getIndex().toString(), ex)) + ); + + if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { + logger.info("TRYING FILE UPLOAD"); + + return () -> { + logger.info("Going to upload {}", indexRouting.prettyPrint()); + + uploadIndex(indexRouting, fileName , blobContainer); + logger.info("upload done {}", indexRouting.prettyPrint()); + + completionListener.onResponse(null); + logger.info("response done {}", indexRouting.prettyPrint()); + + }; } - logger.info("uploadedIndices {}", uploadedIndices); - return uploadedIndices; + logger.info("TRYING S3 UPLOAD"); + + //TODO: Integrate with S3AsyncCrtClient for using buffered stream directly with putObject. + try ( + InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting); + IndexInput input = new ByteArrayIndexInput("indexrouting", indexRoutingStream.readAllBytes())) { + long expectedChecksum; + try { + expectedChecksum = checksumOfChecksum(input.clone(), 8); + } catch (Exception e) { + throw e; + } + try ( + RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + fileName, + fileName, + input.length(), + true, + WritePriority.URGENT, + (size, position) -> new OffsetRangeIndexInputStream(input, size, position), + expectedChecksum, + ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported() + ) + ) { + return () -> ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), completionListener); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } } - public List writeIncrementalRoutingTable( - ClusterState previousClusterState, - ClusterState clusterState, - ClusterMetadataManifest previousManifest) { + public List getAllUploadedIndicesRouting(ClusterMetadataManifest previousManifest, List indicesRoutingToUpload, Set indicesRoutingToDelete) { final Map allUploadedIndicesRouting = previousManifest.getIndicesRouting() .stream() .collect(Collectors.toMap(ClusterMetadataManifest.UploadedIndexMetadata::getIndexName, Function.identity())); + + indicesRoutingToUpload.forEach( + uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) + ); + + indicesRoutingToDelete.forEach(index -> allUploadedIndicesRouting.remove(index)); + logger.info("allUploadedIndicesRouting ROUTING {}", allUploadedIndicesRouting); - Map previousIndexRoutingTable = previousClusterState.getRoutingTable().getIndicesRouting(); - List uploadedIndices = new ArrayList<>(); - BlobPath custerMetadataBasePath = getCusterMetadataBasePath(blobStoreRepository, clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID()); - for (IndexRoutingTable indexRouting : clusterState.getRoutingTable().getIndicesRouting().values()) { - if (previousIndexRoutingTable.containsKey(indexRouting.getIndex().getName()) && indexRouting.equals(previousIndexRoutingTable.get(indexRouting.getIndex().getName()))) { - logger.info("index exists {}", indexRouting.getIndex().getName()); - //existing index with no shard change. - uploadedIndices.add(allUploadedIndicesRouting.get(indexRouting.getIndex().getName())); - } else { - // new index or shards changed, in both cases we upload new index file. - uploadedIndices.add(uploadIndex(indexRouting, custerMetadataBasePath)); - } - } - return uploadedIndices; + return new ArrayList<>(allUploadedIndicesRouting.values()); } - private ClusterMetadataManifest.UploadedIndexMetadata uploadIndex(IndexRoutingTable indexRouting, BlobPath custerMetadataBasePath) { + private void uploadIndex(IndexRoutingTable indexRouting, String fileName, BlobContainer container) { + logger.info("Starting write"); + try { InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting); - BlobContainer container = blobStoreRepository.blobStore().blobContainer(custerMetadataBasePath.add(INDEX_ROUTING_PATH_TOKEN).add(indexRouting.getIndex().getUUID())); - String indexRoutingFileName = getIndexRoutingFileName(); - container.writeBlob(indexRoutingFileName, indexRoutingStream, 4096, true); - return new ClusterMetadataManifest.UploadedIndexMetadata(indexRouting.getIndex().getName(), indexRouting.getIndex().getUUID(), container.path().buildAsString() + indexRoutingFileName); - + container.writeBlob(fileName, indexRoutingStream, 4096, true); + logger.info("SUccessful write"); } catch (IOException e) { logger.error("Failed to write {}", e); } - logger.info("SUccessful write"); - return null; } private String getIndexRoutingFileName() { return String.join( DELIMITER, - //RemoteStoreUtils.invertLong(indexMetadata.getVersion()), - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf("CODEC1") // Keep the codec version at last place only, during read we reads last - // place to determine codec version. + INDEX_ROUTING_FILE_PREFIX, + RemoteStoreUtils.invertLong(System.currentTimeMillis()) ); } - public RoutingTable getLatestRoutingTable(String clusterName, String clusterUUID) { - return null; - } - public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest previousManifest, String clusterName, String clusterUUID) { - return null; - } public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest manifest){ List indicesRoutingDeleted = manifest.getDiffManifest().getIndicesRoutingDeleted(); @@ -232,9 +289,10 @@ public CheckedRunnable getAsyncIndexMetadataReadAction( String uploadedFilename, Index index, LatchedActionListener latchedActionListener) { - BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(INDEX_ROUTING_PATH_TOKEN).add(index.getUUID())); - String[] fileNameTokens = uploadedFilename.split("/"); - String blobFileName = fileNameTokens[fileNameTokens.length -1]; + int idx = uploadedFilename.lastIndexOf("/"); + String blobFileName = uploadedFilename.substring(idx+1); + BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer( BlobPath.cleanPath().add(uploadedFilename.substring(0,idx))); + return () -> readAsync( blobContainer, blobFileName, @@ -262,8 +320,6 @@ public IndexRoutingTableInputStreamReader read(BlobContainer blobContainer, Stri } return null; } - private void deleteStaleRoutingTable(String clusterName, String clusterUUID, int manifestsToRetain) { - } @Override public void close() throws IOException { @@ -300,4 +356,5 @@ public IndexRoutingTable getIndexRoutingTable() { return indexRoutingTable; } } + } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 1ace3a7b3b1f5..00b58335b640e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -947,6 +947,7 @@ public static class UploadedIndexMetadata implements UploadedMetadata, Writeable private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name"); private static final ParseField INDEX_UUID_FIELD = new ParseField("index_uuid"); private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename"); + private static final ParseField COMPONENT_PREFIX_FIELD = new ParseField("component_prefix"); private static String indexName(Object[] fields) { return (String) fields[0]; @@ -960,23 +961,35 @@ private static String uploadedFilename(Object[] fields) { return (String) fields[2]; } + private static String componentPrefix(Object[] fields) { + return (String) fields[3]; + } + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "uploaded_index_metadata", - fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields)) + fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields)) ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_NAME_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_UUID_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), UPLOADED_FILENAME_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD); } static final String COMPONENT_PREFIX = "index--"; + private final String componentPrefix; private final String indexName; private final String indexUUID; private final String uploadedFilename; public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName) { + this( indexName,indexUUID,uploadedFileName, COMPONENT_PREFIX); + } + + public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) { + logger.info("creating UploadedIndexMetadata {}", componentPrefix); + this.componentPrefix = componentPrefix; this.indexName = indexName; this.indexUUID = indexUUID; this.uploadedFilename = uploadedFileName; @@ -986,6 +999,7 @@ public UploadedIndexMetadata(StreamInput in) throws IOException { this.indexName = in.readString(); this.indexUUID = in.readString(); this.uploadedFilename = in.readString(); + this.componentPrefix = in.readString(); } public String getUploadedFilePath() { @@ -994,7 +1008,7 @@ public String getUploadedFilePath() { @Override public String getComponent() { - return COMPONENT_PREFIX + getIndexName(); + return componentPrefix + getIndexName(); } public String getUploadedFilename() { @@ -1010,12 +1024,18 @@ public String getIndexUUID() { return indexUUID; } + public String getComponentPrefix() { + return componentPrefix; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder .field(INDEX_NAME_FIELD.getPreferredName(), getIndexName()) .field(INDEX_UUID_FIELD.getPreferredName(), getIndexUUID()) - .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()); + .field(UPLOADED_FILENAME_FIELD.getPreferredName(), getUploadedFilePath()) + .field(COMPONENT_PREFIX_FIELD.getPreferredName(), getComponentPrefix()); + } @Override @@ -1023,6 +1043,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(indexName); out.writeString(indexUUID); out.writeString(uploadedFilename); + out.writeString(componentPrefix); } @Override @@ -1036,12 +1057,14 @@ public boolean equals(Object o) { final UploadedIndexMetadata that = (UploadedIndexMetadata) o; return Objects.equals(indexName, that.indexName) && Objects.equals(indexUUID, that.indexUUID) - && Objects.equals(uploadedFilename, that.uploadedFilename); + && Objects.equals(uploadedFilename, that.uploadedFilename) + && Objects.equals(componentPrefix, that.componentPrefix); + } @Override public int hashCode() { - return Objects.hash(indexName, indexUUID, uploadedFilename); + return Objects.hash(indexName, indexUUID, uploadedFilename, componentPrefix); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c792a78d1477e..98717c3f7b9fe 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -60,6 +60,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; @@ -186,7 +189,7 @@ public RemoteClusterStateService( if(isRemoteRoutingTableEnabled(settings)) { this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, - settings, clusterSettings, threadPool); + settings, threadPool); logger.info("REMOTE ROUTING ENABLED"); } else { logger.info("REMOTE ROUTING DISABLED"); @@ -212,7 +215,6 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri return null; } - UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel( clusterState, new ArrayList<>(clusterState.metadata().indices().values()), @@ -222,14 +224,8 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true, true, true, - true - ); - - List routingIndexMetadata = new ArrayList<>(); - if(remoteRoutingTableService!=null) { - routingIndexMetadata = remoteRoutingTableService.writeFullRoutingTable(clusterState, previousClusterUUID); - logger.info("routingIndexMetadata {}", routingIndexMetadata); - } + true, + new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values())); final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, @@ -242,7 +238,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedDiscoveryNodes, uploadedMetadataResults.uploadedClusterBlocks, new ClusterStateDiffManifest(clusterState, ClusterState.EMPTY_STATE), - routingIndexMetadata, + uploadedMetadataResults.uploadedIndicesRoutingMetadata, false ); @@ -326,6 +322,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( // index present in current cluster state indicesToBeDeletedFromRemote.remove(indexMetadata.getIndex().getName()); } + + List indicesRoutingToUpload = new ArrayList<>(); + if(remoteRoutingTableService!=null) { + indicesRoutingToUpload = remoteRoutingTableService.getChangedIndicesRouting(previousClusterState, clusterState); + } UploadedMetadataResults uploadedMetadataResults; // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, // If file is empty and codec is 1 then write global metadata. @@ -362,14 +363,10 @@ public ClusterMetadataManifest writeIncrementalMetadata( updateSettingsMetadata, updateTemplatesMetadata, updateDiscoveryNodes, - updateClusterBlocks - ); + updateClusterBlocks, + indicesRoutingToUpload + ); - List routingIndexMetadata = new ArrayList<>(); - if(remoteRoutingTableService!=null) { - routingIndexMetadata = remoteRoutingTableService.writeIncrementalRoutingTable(previousClusterState, clusterState, previousManifest); - logger.info("routingIndexMetadata incremental {}", routingIndexMetadata); - } // update the map if the metadata was uploaded uploadedMetadataResults.uploadedIndexMetadata.forEach( @@ -380,6 +377,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( customsToBeDeletedFromRemote.keySet().forEach(allUploadedCustomMap::remove); indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove); + List allUploadedIndicesRouting = new ArrayList<>(); + if(remoteRoutingTableService!=null) { + allUploadedIndicesRouting = remoteRoutingTableService.getAllUploadedIndicesRouting(previousManifest, uploadedMetadataResults.uploadedIndicesRoutingMetadata, indicesToBeDeletedFromRemote.keySet()); + } + final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, new ArrayList<>(allUploadedIndexMetadata.values()), @@ -393,8 +395,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( firstUploadForSplitGlobalMetadata || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), firstUploadForSplitGlobalMetadata || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), new ClusterStateDiffManifest(clusterState, previousClusterState), - routingIndexMetadata, false + allUploadedIndicesRouting, false ); + + logger.info("MANIFEST IN INC STATE {}", manifest); + this.latestClusterName = clusterState.getClusterName().value(); this.latestClusterUUID = clusterState.metadata().clusterUUID(); @@ -459,11 +464,10 @@ private UploadedMetadataResults writeMetadataInParallel( boolean uploadSettingsMetadata, boolean uploadTemplateMetadata, boolean uploadDiscoveryNodes, - boolean uploadClusterBlock - ) throws IOException { - assert Objects.nonNull(indexMetadataUploadListeners) : "indexMetadataUploadListeners can not be null"; + boolean uploadClusterBlock, + List indicesRoutingToUpload) throws IOException { int totalUploadTasks = indexToUpload.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata - ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0); + ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size(); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new HashMap<>(totalUploadTasks); Map results = new HashMap<>(totalUploadTasks); @@ -471,7 +475,7 @@ private UploadedMetadataResults writeMetadataInParallel( LatchedActionListener listener = new LatchedActionListener<>( ActionListener.wrap((ClusterMetadataManifest.UploadedMetadata uploadedMetadata) -> { - logger.trace(String.format(Locale.ROOT, "Metadata component %s uploaded successfully.", uploadedMetadata.getComponent())); + logger.info(String.format(Locale.ROOT, "Metadata component %s uploaded successfully.", uploadedMetadata.getComponent())); results.put(uploadedMetadata.getComponent(), uploadedMetadata); }, ex -> { logger.error( @@ -563,6 +567,17 @@ private UploadedMetadataResults writeMetadataInParallel( ); }); + indicesRoutingToUpload.forEach(indexRoutingTable -> { + try { + uploadTasks.put( + indexRoutingTable.getIndex().getName() + "--indexRouting", + remoteRoutingTableService.getIndexRoutingAsyncAction(clusterState, indexRoutingTable, listener) + ); + } catch (IOException e) { + e.printStackTrace(); + } + }); + // start async upload of all required metadata files for (CheckedRunnable uploadTask : uploadTasks.values()) { uploadTask.run(); @@ -608,7 +623,10 @@ private UploadedMetadataResults writeMetadataInParallel( } UploadedMetadataResults response = new UploadedMetadataResults(); results.forEach((name, uploadedMetadata) -> { - if (name.contains(CUSTOM_METADATA)) { + if (uploadedMetadata.getClass().equals(UploadedIndexMetadata.class) && + uploadedMetadata.getComponent().contains(RemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX)) { + response.uploadedIndicesRoutingMetadata.add((UploadedIndexMetadata) uploadedMetadata); + } else if (name.contains(CUSTOM_METADATA)) { // component name for custom metadata will look like custom-- String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; response.uploadedCustomMetadataMap.put( @@ -631,6 +649,7 @@ private UploadedMetadataResults writeMetadataInParallel( throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); } }); + logger.info("response {}", response.uploadedIndicesRoutingMetadata.toString()); return response; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index abedd23c1e742..1574fc935058a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -80,6 +80,7 @@ public static class UploadedMetadataResults { ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes; ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks; + List uploadedIndicesRoutingMetadata; public UploadedMetadataResults( List uploadedIndexMetadata, @@ -88,7 +89,8 @@ public UploadedMetadataResults( ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks, + List uploadedIndicesRoutingMetadata ) { this.uploadedIndexMetadata = uploadedIndexMetadata; this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; @@ -97,6 +99,7 @@ public UploadedMetadataResults( this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; this.uploadedDiscoveryNodes = uploadedDiscoveryNodes; this.uploadedClusterBlocks = uploadedClusterBlocks; + this.uploadedIndicesRoutingMetadata = uploadedIndicesRoutingMetadata; } public UploadedMetadataResults() { @@ -107,6 +110,7 @@ public UploadedMetadataResults() { this.uploadedTemplatesMetadata = null; this.uploadedDiscoveryNodes = null; this.uploadedClusterBlocks = null; + this.uploadedIndicesRoutingMetadata = new ArrayList<>(); } } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index c1ac74724e405..78361079c9176 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -111,13 +111,20 @@ boolean requiresHashAlgorithm() { @Override public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; - return BlobPath.cleanPath() + BlobPath path = BlobPath.cleanPath() .add(hashAlgorithm.hash(pathInput)) .add(pathInput.basePath()) - .add(pathInput.indexUUID()) - .add(pathInput.shardId()) - .add(pathInput.dataCategory().getName()) - .add(pathInput.dataType().getName()); + .add(pathInput.indexUUID()); + if (pathInput.shardId() != null) { + path.add(pathInput.shardId()); + } + if(pathInput.dataCategory() != null){ + path.add(pathInput.dataCategory().getName()); + } + if(pathInput.dataType() != null ) { + path.add(pathInput.dataType().getName()); + } + return path; } @Override @@ -188,11 +195,11 @@ public static PathType fromCode(int code) { public BlobPath path(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { DataCategory dataCategory = pathInput.dataCategory(); DataType dataType = pathInput.dataType(); - assert dataCategory.isSupportedDataType(dataType) : "category:" - + dataCategory - + " type:" - + dataType - + " are not supported together"; +// assert dataCategory.isSupportedDataType(dataType) : "category:" +// + dataCategory +// + " type:" +// + dataType +// + " are not supported together"; return generatePath(pathInput, hashAlgorithm); } @@ -227,8 +234,7 @@ public enum PathHashAlgorithm { FNV_1A_BASE64(0) { @Override String hash(PathInput pathInput) { - String input = pathInput.indexUUID() + pathInput.shardId() + pathInput.dataCategory().getName() + pathInput.dataType() - .getName(); + String input = pathInput.indexUUID() + pathInput.shardId() + (pathInput.dataCategory() != null ? pathInput.dataCategory().getName(): "" )+ (pathInput.dataType()!= null ? pathInput.dataType().getName(): ""); long hash = FNV1a.hash64(input); return longToUrlBase64(hash); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java index c58f6c3faac84..b5f28e4b6fdee 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java @@ -87,9 +87,12 @@ public static class PathInput { public PathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) { this.basePath = Objects.requireNonNull(basePath); this.indexUUID = Objects.requireNonNull(indexUUID); - this.shardId = Objects.requireNonNull(shardId); - this.dataCategory = Objects.requireNonNull(dataCategory); - this.dataType = Objects.requireNonNull(dataType); +// this.shardId = Objects.requireNonNull(shardId); +// this.dataCategory = Objects.requireNonNull(dataCategory); +// this.dataType = Objects.requireNonNull(dataType); + this.shardId =(shardId); + this.dataCategory = (dataCategory); + this.dataType = (dataType); } BlobPath basePath() { From 8f0902c5b37883617b0fb3ee27557065381bd870 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 23 May 2024 13:25:16 +0530 Subject: [PATCH 077/133] Add metadata version and change codec version to V3 Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 115 ++++++++---------- .../gateway/remote/RemoteManifestManager.java | 48 ++++---- 2 files changed, 75 insertions(+), 88 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 00b58335b640e..4b3c86207d0fa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -43,8 +43,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. - public static final int CODEC_V3 = 3; // In Codec V3, we introduced diff as part of manifest. - public static final int CODEC_V4 = 4; // In Codec V4, we introduce index routing-metadata in manifest file. + public static final int CODEC_V3 = 3; // In Codec V3, we introduced index routing-metadata, diff as part of manifest. private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -58,6 +57,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid"); private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed"); + private static final ParseField METADATA_VERSION = new ParseField("metadata_version"); private static final ParseField UPLOADED_COORDINATOR_METADATA = new ParseField("uploaded_coordinator_metadata"); private static final ParseField UPLOADED_SETTINGS_METADATA = new ParseField("uploaded_settings_metadata"); private static final ParseField UPLOADED_TEMPLATES_METADATA = new ParseField("uploaded_templates_metadata"); @@ -95,6 +95,16 @@ private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields .customMetadataMap(customMetadata(fields)); } + private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields) { + return manifestV2Builder(fields) + .discoveryNodesMetadata(discoveryNodesMetadata(fields)) + .clusterBlocksMetadata(clusterBlocksMetadata(fields)) + .diffManifest(diffManifest(fields)) + .routingTableVersion(routingTableVersion(fields)) + .indicesRouting(indicesRouting(fields)) + .metadataVersion(metadataVersion(fields)); + } + private static long term(Object[] fields) { return (long) fields[0]; } @@ -180,6 +190,10 @@ private static List indicesRouting(Object[] fields) { return (List) fields[19]; } + private static long metadataVersion(Object[] fields) { + return (long) fields[20]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> manifestV0Builder(fields).build() @@ -197,62 +211,16 @@ private static List indicesRouting(Object[] fields) { private static final ConstructingObjectParser PARSER_V3 = new ConstructingObjectParser<>( "cluster_metadata_manifest", - fields -> ClusterMetadataManifest.builder() - .clusterTerm(term(fields)) - .stateVersion(version(fields)) - .clusterUUID(clusterUUID(fields)) - .stateUUID(stateUUID(fields)) - .opensearchVersion(opensearchVersion(fields)) - .nodeId(nodeId(fields)) - .committed(committed(fields)) - .codecVersion(codecVersion(fields)) - .indices(indices(fields)) - .previousClusterUUID(previousClusterUUID(fields)) - .clusterUUIDCommitted(clusterUUIDCommitted(fields)) - .coordinationMetadata(coordinationMetadata(fields)) - .settingMetadata(settingsMetadata(fields)) - .templatesMetadata(templatesMetadata(fields)) - .customMetadataMap(customMetadata(fields)) - .discoveryNodesMetadata(discoveryNodesMetadata(fields)) - .clusterBlocksMetadata(clusterBlocksMetadata(fields)) - .diffManifest(diffManifest(fields)) - .build() + fields -> manifestV3Builder(fields).build() ); - private static final ConstructingObjectParser PARSER_V4 = new ConstructingObjectParser<>( - "cluster_metadata_manifest", - fields -> ClusterMetadataManifest.builder() - .clusterTerm(term(fields)) - .stateVersion(version(fields)) - .clusterUUID(clusterUUID(fields)) - .stateUUID(stateUUID(fields)) - .opensearchVersion(opensearchVersion(fields)) - .nodeId(nodeId(fields)) - .committed(committed(fields)) - .codecVersion(codecVersion(fields)) - .indices(indices(fields)) - .previousClusterUUID(previousClusterUUID(fields)) - .clusterUUIDCommitted(clusterUUIDCommitted(fields)) - .coordinationMetadata(coordinationMetadata(fields)) - .settingMetadata(settingsMetadata(fields)) - .templatesMetadata(templatesMetadata(fields)) - .customMetadataMap(customMetadata(fields)) - .discoveryNodesMetadata(discoveryNodesMetadata(fields)) - .clusterBlocksMetadata(clusterBlocksMetadata(fields)) - .diffManifest(diffManifest(fields)) - .routingTableVersion(routingTableVersion(fields)) - .indicesRouting(indicesRouting(fields)) - .build() - ); - - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V4; + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V3; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); declareParser(PARSER_V2, CODEC_V2); declareParser(PARSER_V3, CODEC_V3); - declareParser(PARSER_V4, CODEC_V4); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -313,19 +281,19 @@ private static void declareParser(ConstructingObjectParser ClusterStateDiffManifest.fromXContent(p), DIFF_MANIFEST ); - } - if (codec_version >= CODEC_V4) { parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); parser.declareObjectArray( ConstructingObjectParser.constructorArg(), (p, c) -> UploadedIndexMetadata.fromXContent(p), INDICES_ROUTING_FIELD ); + parser.declareLong(ConstructingObjectParser.constructorArg(), METADATA_VERSION); } } private final int codecVersion; private final String globalMetadataFileName; + private final long metadataVersion; private final UploadedMetadataAttribute uploadedCoordinationMetadata; private final UploadedMetadataAttribute uploadedSettingsMetadata; private final UploadedMetadataAttribute uploadedTemplatesMetadata; @@ -395,6 +363,11 @@ public String getGlobalMetadataFileName() { } private static final Logger logger = LogManager.getLogger(ClusterMetadataManifest.class); + + public long getMetadataVersion() { + return metadataVersion; + } + public UploadedMetadataAttribute getCoordinationMetadata() { return uploadedCoordinationMetadata; } @@ -458,7 +431,8 @@ public ClusterMetadataManifest( UploadedMetadataAttribute clusterBlocksMetadata, ClusterStateDiffManifest diffManifest, long routingTableVersion, - List indicesRouting + List indicesRouting, + long metadataVersion ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -483,6 +457,7 @@ public ClusterMetadataManifest( this.diffManifest = diffManifest; this.routingTableVersion = routingTableVersion; this.indicesRouting = Collections.unmodifiableList(indicesRouting); + this.metadataVersion = metadataVersion; } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -512,6 +487,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.diffManifest = null; this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); + this.metadataVersion = in.readLong(); } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); @@ -524,6 +500,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedDiscoveryNodesMetadata = null; this.uploadedClusterBlocksMetadata = null; this.diffManifest = null; + this.metadataVersion = -1; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; @@ -536,6 +513,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedDiscoveryNodesMetadata = null; this.uploadedClusterBlocksMetadata = null; this.diffManifest = null; + this.metadataVersion = -1; } } @@ -585,6 +563,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws getTemplatesMetadata().toXContent(builder, params); builder.endObject(); } + builder.startObject(UPLOADED_CUSTOM_METADATA.getPreferredName()); + for (UploadedMetadataAttribute attribute : getCustomMetadataMap().values()) { + attribute.toXContent(builder, params); + } + builder.endObject(); + } else if (onOrAfterCodecVersion(CODEC_V1)) { + builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); + builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); + } + if (onOrAfterCodecVersion(CODEC_V3)) { if (getDiscoveryNodesMetadata() != null) { builder.startObject(UPLOADED_DISCOVERY_NODES_METADATA.getPreferredName()); getDiscoveryNodesMetadata().toXContent(builder, params); @@ -600,16 +588,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws getDiffManifest().toXContent(builder, params); builder.endObject(); } - builder.startObject(UPLOADED_CUSTOM_METADATA.getPreferredName()); - for (UploadedMetadataAttribute attribute : getCustomMetadataMap().values()) { - attribute.toXContent(builder, params); - } - builder.endObject(); - } else if (onOrAfterCodecVersion(CODEC_V1)) { - builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); - builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); - } - if (onOrAfterCodecVersion(CODEC_V4)) { + builder.field(METADATA_VERSION.getPreferredName(), getMetadataVersion()); builder.field(ROUTING_TABLE_VERSION_FIELD.getPreferredName(), getRoutingTableVersion()); builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { @@ -644,6 +623,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(uploadedCustomMetadataMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); + out.writeLong(metadataVersion); } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); @@ -724,6 +704,7 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws public static class Builder { private String globalMetadataFileName; + private long metadataVersion; private UploadedMetadataAttribute discoveryNodesMetadata; private UploadedMetadataAttribute clusterBlocksMetadata; private UploadedMetadataAttribute coordinationMetadata; @@ -755,6 +736,11 @@ public Builder routingTableVersion(long routingTableVersion) { return this; } + public Builder metadataVersion(long metadataVersion) { + this.metadataVersion = metadataVersion; + return this; + } + public Builder indicesRouting(List indicesRouting) { this.indicesRouting = indicesRouting; return this; @@ -912,7 +898,8 @@ public ClusterMetadataManifest build() { clusterBlocksMetadata, diffManifest, routingTableVersion, - indicesRouting + indicesRouting, + metadataVersion ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index c1e0ad4ece47a..f597e43149a07 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -44,7 +44,7 @@ public class RemoteManifestManager { public static final String MANIFEST_PATH_TOKEN = "manifest"; public static final String MANIFEST_FILE_PREFIX = "manifest"; public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V4; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); @@ -141,29 +141,29 @@ ClusterMetadataManifest uploadManifest( committed, MANIFEST_CURRENT_CODEC_VERSION ); - final ClusterMetadataManifest manifest = new ClusterMetadataManifest( - clusterState.term(), - clusterState.getVersion(), - clusterState.metadata().clusterUUID(), - clusterState.stateUUID(), - Version.CURRENT, - nodeId, - committed, - MANIFEST_CURRENT_CODEC_VERSION, - null, - uploadedIndexMetadata, - previousClusterUUID, - clusterState.metadata().clusterUUIDCommitted(), - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplatesMetadata, - uploadedCustomMetadataMap, - uploadedDiscoveryNodesMetadata, - uploadedClusterBlocksMetadata, - clusterDiffManifest, - clusterState.getRoutingTable().version(), - routingIndexMetadata - ); + ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder(); + manifestBuilder.clusterTerm(clusterState.term()) + .stateVersion(clusterState.getVersion()) + .clusterUUID(clusterState.metadata().clusterUUID()) + .stateUUID(clusterState.stateUUID()) + .opensearchVersion(Version.CURRENT) + .nodeId(nodeId) + .committed(committed) + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .indices(uploadedIndexMetadata) + .previousClusterUUID(previousClusterUUID) + .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) + .coordinationMetadata(uploadedCoordinationMetadata) + .settingMetadata(uploadedSettingsMetadata) + .templatesMetadata(uploadedTemplatesMetadata) + .customMetadataMap(uploadedCustomMetadataMap) + .discoveryNodesMetadata(uploadedDiscoveryNodesMetadata) + .clusterBlocksMetadata(uploadedClusterBlocksMetadata) + .diffManifest(clusterDiffManifest) + .routingTableVersion(clusterState.getRoutingTable().version()) + .indicesRouting(routingIndexMetadata) + .metadataVersion(clusterState.metadata().version()); + final ClusterMetadataManifest manifest = manifestBuilder.build(); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; } From 7d630e22cd62df9e6685b89d515caa77224aab76 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 23 May 2024 15:43:55 +0530 Subject: [PATCH 078/133] Minor Refactoring for RemoteReadResults Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 61 +------------------ .../RemoteClusterStateAttributesManager.java | 14 +---- .../remote/RemoteClusterStateService.java | 20 ++---- .../remote/RemoteClusterStateUtils.java | 24 -------- .../remote/RemoteGlobalMetadataManager.java | 4 +- .../remote/RemoteIndexMetadataManager.java | 4 +- .../gateway/remote/RemoteReadResult.java | 35 +++++++++++ .../remotestore/RemoteStoreNodeAttribute.java | 11 ++-- 8 files changed, 54 insertions(+), 119 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteReadResult.java diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 499296a15e6b6..170fe44554d27 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -14,7 +14,6 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; @@ -29,8 +28,10 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.gateway.remote.RemoteReadResult; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStream; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStreamReader; import org.opensearch.index.remote.RemoteStoreEnums; @@ -48,7 +49,6 @@ import java.io.InputStream; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -78,6 +78,7 @@ public class RemoteRoutingTableService implements Closeable { Setting.Property.Final ); public static final String INDEX_ROUTING_PATH_TOKEN = "index-routing"; + public static final String ROUTING_TABLE = "routing-table"; public static final String INDEX_ROUTING_FILE_PREFIX = "index_routing"; public static final String DELIMITER = "__"; public static final String INDEX_ROUTING_METADATA_PREFIX = "indexRouting--"; @@ -229,63 +230,7 @@ private String getIndexRoutingFileName() { } - - public RoutingTable getIncrementalRoutingTable(ClusterState previousClusterState, ClusterMetadataManifest manifest){ - List indicesRoutingDeleted = manifest.getDiffManifest().getIndicesRoutingDeleted(); - List indicesRoutingUpdated = manifest.getDiffManifest().getIndicesRoutingUpdated(); - - List indicesRoutingUpdatedMetadata = manifest.getIndicesRouting().stream() - .filter(indexRouting -> indicesRoutingUpdated.contains(indexRouting.getIndexName())) - .collect(Collectors.toList()); - - Map indicesRouting = new HashMap<>(previousClusterState.getRoutingTable().indicesRouting()); - indicesRoutingDeleted.forEach(indicesRouting::remove); - - for(ClusterMetadataManifest.UploadedIndexMetadata indexRoutingMetaData: indicesRoutingUpdatedMetadata) { - logger.debug("Starting the read for first indexRoutingMetaData: {}", indexRoutingMetaData); - String filePath = indexRoutingMetaData.getUploadedFilePath(); - BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath()); - try { - InputStream inputStream = container.readBlob(filePath); - IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); - Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); - IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); - indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); - logger.debug("IndexRouting {}", indexRouting); - } catch (IOException e) { - logger.info("RoutingTable read failed with error: {}", e.toString()); - } - - } - return new RoutingTable(manifest.getRoutingTableVersion(), indicesRouting); - } - - public RoutingTable getFullRoutingTable(long routingTableVersion, List indicesRoutingMetaData) { - Map indicesRouting = new HashMap<>(); - - for(ClusterMetadataManifest.UploadedIndexMetadata indexRoutingMetaData: indicesRoutingMetaData) { - logger.debug("Starting the read for first indexRoutingMetaData: {}", indexRoutingMetaData); - String filePath = indexRoutingMetaData.getUploadedFilePath(); - BlobContainer container = blobStoreRepository.blobStore().blobContainer(blobStoreRepository.basePath().add(filePath)); - - try { - InputStream inputStream = container.readBlob(indexRoutingMetaData.getIndexName()); - IndexRoutingTableInputStreamReader indexRoutingTableInputStreamReader = new IndexRoutingTableInputStreamReader(inputStream); - Index index = new Index(indexRoutingMetaData.getIndexName(), indexRoutingMetaData.getIndexUUID()); - IndexRoutingTable indexRouting = indexRoutingTableInputStreamReader.readIndexRoutingTable(index); - indicesRouting.put(indexRoutingMetaData.getIndexName(), indexRouting); - logger.debug("IndexRouting {}", indexRouting); - } catch (IOException e) { - logger.info("RoutingTable read failed with error: {}", e.toString()); - } - - } - return new RoutingTable(routingTableVersion, indicesRouting); - } - public CheckedRunnable getAsyncIndexMetadataReadAction( - String clusterName, - String clusterUUID, String uploadedFilename, Index index, LatchedActionListener latchedActionListener) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index ae1f1ecc36cde..5c7eb4cc25fee 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -8,30 +8,20 @@ package org.opensearch.gateway.remote; -import java.util.Locale; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlocks; -import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; - public class RemoteClusterStateAttributesManager { public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; public static final String DISCOVERY_NODES = "nodes"; @@ -83,10 +73,10 @@ public CheckedRunnable getAsyncMetadataReadAction( String clusterUUID, String component, String uploadedFilename, - LatchedActionListener listener + LatchedActionListener listener ) { AbstractRemoteBlobStoreObject remoteObject = getRemoteObject(component, uploadedFilename, clusterUUID); - ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); + ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); return () -> remoteObject.readAsync(actionListener); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 2ea5ae9edf6a8..376efe97c5272 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -13,7 +13,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteReadResult; + import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; @@ -33,7 +33,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -58,10 +57,7 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.cluster.node.DiscoveryNodes; -import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -842,10 +838,9 @@ private ClusterState readClusterStateInParallel( List> asyncMetadataReadActions = new ArrayList<>(); List readResults = new ArrayList<>(); List readIndexRoutingTableResults = new ArrayList<>(); - List indexRoutingExceptionList = Collections.synchronizedList(new ArrayList<>(indicesRoutingToRead.size())); List exceptionList = Collections.synchronizedList(new ArrayList<>(totalReadTasks)); - LatchedActionListener listener = new LatchedActionListener<>( + LatchedActionListener listener = new LatchedActionListener<>( ActionListener.wrap( response -> { logger.info("Successfully read cluster state component from remote"); @@ -877,7 +872,7 @@ private ClusterState readClusterStateInParallel( }, ex -> { logger.error("Failed to read cluster state from remote", ex); - indexRoutingExceptionList.add(ex); + exceptionList.add(ex); } ), latch @@ -886,8 +881,6 @@ private ClusterState readClusterStateInParallel( for (UploadedIndexMetadata indexRouting: indicesRoutingToRead) { asyncMetadataReadActions.add( remoteRoutingTableService.getAsyncIndexMetadataReadAction( - clusterName, - clusterUUID, indexRouting.getUploadedFilename(), new Index(indexRouting.getIndexName(), indexRouting.getIndexUUID()), routingTableLatchedActionListener @@ -994,18 +987,13 @@ private ClusterState readClusterStateInParallel( throw exception; } - if (!indexRoutingExceptionList.isEmpty()) { - RemoteStateTransferException exception = new RemoteStateTransferException("Exception during reading routing table from remote"); - exceptionList.forEach(exception::addSuppressed); - throw exception; - } ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); AtomicReference discoveryNodesBuilder = new AtomicReference<>(DiscoveryNodes.builder()); Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); metadataBuilder.clusterUUID(manifest.getClusterUUID()); metadataBuilder.clusterUUIDCommitted(manifest.isClusterUUIDCommitted()); Map indexMetadataMap = new HashMap<>(); - Map indicesRouting = new HashMap<>(); + Map indicesRouting = new HashMap<>(previousState.routingTable().getIndicesRouting()); readResults.forEach(remoteReadResult -> { switch (remoteReadResult.getComponent()) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 1574fc935058a..a383291637a9e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -113,28 +113,4 @@ public UploadedMetadataResults() { this.uploadedIndicesRoutingMetadata = new ArrayList<>(); } } - - public static class RemoteReadResult { - ToXContent obj; - String component; - String componentName; - - public RemoteReadResult(ToXContent obj, String component, String componentName) { - this.obj = obj; - this.component = component; - this.componentName = componentName; - } - - public ToXContent getObj() { - return obj; - } - - public String getComponent() { - return component; - } - - public String getComponentName() { - return componentName; - } - } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 5ad7c2de6b5e3..96aa8dc8bdaf6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -125,7 +125,7 @@ CheckedRunnable getAsyncMetadataReadAction( String component, String componentName, String uploadFilename, - LatchedActionListener listener + LatchedActionListener listener ) { AbstractRemoteBlobStoreObject remoteBlobStoreObject; if (component.equals(COORDINATION_METADATA)) { @@ -139,7 +139,7 @@ CheckedRunnable getAsyncMetadataReadAction( } else { throw new RemoteStateTransferException("Unknown component " + componentName); } - ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteClusterStateUtils.RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); + ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); return () -> remoteBlobStoreObject.readAsync(actionListener); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index 09df6248abeb6..fc90ba82999ae 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -74,11 +74,11 @@ CheckedRunnable getIndexMetadataAsyncAction(IndexMetadata indexMeta CheckedRunnable getAsyncIndexMetadataReadAction( String clusterUUID, String uploadedFilename, - LatchedActionListener latchedActionListener + LatchedActionListener latchedActionListener ) { RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); ActionListener actionListener = ActionListener.wrap( - response -> latchedActionListener.onResponse(new RemoteClusterStateUtils.RemoteReadResult(response, INDEX_PATH_TOKEN, response.getIndexName())), + response -> latchedActionListener.onResponse(new RemoteReadResult(response, INDEX_PATH_TOKEN, response.getIndexName())), latchedActionListener::onFailure); return () -> remoteIndexMetadata.readAsync(actionListener); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteReadResult.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteReadResult.java new file mode 100644 index 0000000000000..dea8a6221d05a --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteReadResult.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.core.xcontent.ToXContent; + +public class RemoteReadResult { + ToXContent obj; + String component; + String componentName; + + public RemoteReadResult(ToXContent obj, String component, String componentName) { + this.obj = obj; + this.component = component; + this.componentName = componentName; + } + + public ToXContent getObj() { + return obj; + } + + public String getComponent() { + return component; + } + + public String getComponentName() { + return componentName; + } +} diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 04febb61727eb..3a71b74c3fb70 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -115,11 +115,11 @@ private Map validateSettingsAttributesNonNull(DiscoveryNode node .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(node, key))); - if (settingsMap.isEmpty()) { - throw new IllegalStateException( - "joining node [" + node + "] doesn't have settings attribute for [" + repositoryName + "] repository" - ); - } +// if (settingsMap.isEmpty()) { +// throw new IllegalStateException( +// "joining node [" + node + "] doesn't have settings attribute for [" + repositoryName + "] repository" +// ); +// } return settingsMap; } @@ -138,6 +138,7 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na // Repository metadata built here will always be for a system repository. settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); + settings.put("repositories.fs.location", "/Users/abandeji/Public/work-dump/remote/snap"); return new RepositoryMetadata(name, type, settings.build(), cryptoMetadata); } From c1cd20c30f30c065ff661481ec1f24998177ff6c Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 23 May 2024 15:53:56 +0530 Subject: [PATCH 079/133] Set metadata version in cluster state --- .../ChecksumBlobStoreFormatRegistry.java | 18 ------------------ .../remote/RemoteClusterStateService.java | 4 ++-- 2 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java b/server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java deleted file mode 100644 index 5708367f15b56..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/ChecksumBlobStoreFormatRegistry.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote; - -import java.util.Map; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; - -public class ChecksumBlobStoreFormatRegistry { - - public static Map, ChecksumBlobStoreFormat> checksumBlobStoreFormatMap; - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 376efe97c5272..65babd38c645e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -990,6 +990,7 @@ private ClusterState readClusterStateInParallel( ClusterState.Builder clusterStateBuilder = ClusterState.builder(previousState); AtomicReference discoveryNodesBuilder = new AtomicReference<>(DiscoveryNodes.builder()); Metadata.Builder metadataBuilder = Metadata.builder(previousState.metadata()); + metadataBuilder.version(manifest.getMetadataVersion()); metadataBuilder.clusterUUID(manifest.getClusterUUID()); metadataBuilder.clusterUUIDCommitted(manifest.isClusterUUIDCommitted()); Map indexMetadataMap = new HashMap<>(); @@ -1041,9 +1042,8 @@ private ClusterState readClusterStateInParallel( } public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId) throws IOException { - // todo make this async return readClusterStateInParallel( - ClusterState.EMPTY_STATE, + ClusterState.builder(new ClusterName(clusterName)).build(), manifest, clusterName, manifest.getClusterUUID(), From bb8c2081d9785cf22fa709c3f77583796347ae81 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 23 May 2024 16:10:44 +0530 Subject: [PATCH 080/133] remove excessive logging Signed-off-by: Arpit Bandejiya --- .../cluster/coordination/PublicationTransportHandler.java | 3 ++- .../opensearch/gateway/remote/ClusterMetadataManifest.java | 1 - .../opensearch/gateway/remote/RemoteClusterStateService.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 7c83707c5120a..1f1f795139651 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -249,12 +249,13 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish if (applyFullState == true) { ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest, transportService.getLocalNode().getId()); - logger.debug("Downloaded full cluster state version [{}]", clusterState.version()); + logger.debug("Downloaded full cluster state [{}]", clusterState); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.set(clusterState); return response; } else { ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get(), transportService.getLocalNode().getId()); + logger.debug("Downloaded full cluster state from diff [{}]", clusterState); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.compareAndSet(lastSeen, clusterState); return response; diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4b3c86207d0fa..41a400965dac4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -975,7 +975,6 @@ public UploadedIndexMetadata(String indexName, String indexUUID, String uploaded } public UploadedIndexMetadata(String indexName, String indexUUID, String uploadedFileName, String componentPrefix) { - logger.info("creating UploadedIndexMetadata {}", componentPrefix); this.componentPrefix = componentPrefix; this.indexName = indexName; this.indexUUID = indexUUID; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 376efe97c5272..69282fd12b8d6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -843,7 +843,7 @@ private ClusterState readClusterStateInParallel( LatchedActionListener listener = new LatchedActionListener<>( ActionListener.wrap( response -> { - logger.info("Successfully read cluster state component from remote"); + logger.debug("Successfully read cluster state component from remote"); readResults.add(response); }, ex -> { @@ -867,7 +867,7 @@ private ClusterState readClusterStateInParallel( LatchedActionListener routingTableLatchedActionListener = new LatchedActionListener<>( ActionListener.wrap( response -> { - logger.info("Successfully read cluster state component from remote"); + logger.debug("Successfully read cluster state component from remote"); readIndexRoutingTableResults.add(response); }, ex -> { From a9ddeadd0d099eb83e68f57e459375a21d4aad30 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 23 May 2024 16:23:43 +0530 Subject: [PATCH 081/133] Fix build issue Signed-off-by: Arpit Bandejiya --- .../opensearch/node/remotestore/RemoteStoreNodeAttribute.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 3a71b74c3fb70..29eaecf2024de 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -138,7 +138,6 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na // Repository metadata built here will always be for a system repository. settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); - settings.put("repositories.fs.location", "/Users/abandeji/Public/work-dump/remote/snap"); return new RepositoryMetadata(name, type, settings.build(), cryptoMetadata); } From f7fef9769ed768d1312e619c32141e09ff0d25d8 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 23 May 2024 16:26:16 +0530 Subject: [PATCH 082/133] Fixing build --- .../gateway/remote/RemoteClusterStateCleanupManagerTests.java | 1 - .../gateway/remote/RemoteClusterStateServiceTests.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 6d1a8e0207851..d69864a9d2268 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -69,7 +69,6 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteIndexMetadataManager.INDEX_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index acd8e19102b00..04a9efe9ae4ea 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -303,7 +303,7 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { .provideStream(0) .getInputStream() .readAllBytes(); - IndexMetadata writtenIndexMetadata = RemoteIndexMetadataManager.INDEX_METADATA_FORMAT.deserialize( + IndexMetadata writtenIndexMetadata = RemoteIndexMetadata.INDEX_METADATA_FORMAT.deserialize( capturedWriteContext.get("metadata").getFileName(), blobStoreRepository.getNamedXContentRegistry(), new BytesArray(writtenBytes) @@ -1569,7 +1569,7 @@ private void mockBlobContainer( } String fileName = uploadedIndexMetadata.getUploadedFilename(); when(blobContainer.readBlob(fileName + ".dat")).thenAnswer((invocationOnMock) -> { - BytesReference bytesIndexMetadata = RemoteIndexMetadataManager.INDEX_METADATA_FORMAT.serialize( + BytesReference bytesIndexMetadata = RemoteIndexMetadata.INDEX_METADATA_FORMAT.serialize( indexMetadata, fileName, blobStoreRepository.getCompressor(), From ae2ed6ba2ddb9cabcbf7457ae83830b5e9ef0a1c Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 23 May 2024 20:34:44 +0530 Subject: [PATCH 083/133] Add custom metadata diff Signed-off-by: Shivansh Arora --- .../remote/ClusterStateDiffManifest.java | 67 +++++++++++++------ .../remote/RemoteClusterStateService.java | 12 +--- 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 18518de4180df..98dbadbdaefaa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote; +import org.apache.lucene.util.packed.DirectMonotonicReader; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; @@ -35,19 +36,20 @@ public class ClusterStateDiffManifest implements ToXContentObject { private static final String METADATA_DIFF_FIELD = "metadata_diff"; private static final String COORDINATION_METADATA_UPDATED_FIELD = "coordination_metadata_diff"; private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff"; - private static final String TEMPLATES_METADATA_UPDATED_FIELD = ("templates_metadata_diff"); - private static final String INDICES_DIFF_FIELD = ("indices_diff"); - private static final String UPSERTS_FIELD = ("upserts"); - private static final String DELETES_FIELD = ("deletes"); - private static final String CLUSTER_BLOCKS_UPDATED_FIELD = ("cluster_blocks_diff"); - private static final String DISCOVERY_NODES_UPDATED_FIELD = ("discovery_nodes_diff"); - private static final String ROUTING_TABLE_DIFF = ("routing_table_diff"); + private static final String TEMPLATES_METADATA_UPDATED_FIELD = "templates_metadata_diff"; + private static final String INDICES_DIFF_FIELD = "indices_diff"; + private static final String METADATA_CUSTOM_DIFF_FIELD = "metadata_custom_diff"; + private static final String UPSERTS_FIELD = "upserts"; + private static final String DELETES_FIELD = "deletes"; + private static final String CLUSTER_BLOCKS_UPDATED_FIELD = "cluster_blocks_diff"; + private static final String DISCOVERY_NODES_UPDATED_FIELD = "discovery_nodes_diff"; + private static final String ROUTING_TABLE_DIFF = "routing_table_diff"; private final String fromStateUUID; private final String toStateUUID; private final boolean coordinationMetadataUpdated; private final boolean settingsMetadataUpdated; private final boolean templatesMetadataUpdated; - private final Map customMetadataUpdated; + private List customMetadataUpdated; private final List customMetadataDeleted; private final List indicesUpdated; private final List indicesDeleted; @@ -66,12 +68,11 @@ public class ClusterStateDiffManifest implements ToXContentObject { indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); clusterBlocksUpdated = state.blocks().equals(previousState.blocks()); discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); - customMetadataUpdated = new HashMap<>(); + customMetadataUpdated = new ArrayList<>(); for (String custom : state.metadata().customs().keySet()) { - customMetadataUpdated.put( - custom, - state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom)) - ); + if (!state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) { + customMetadataUpdated.add(custom); + } } customMetadataDeleted = new ArrayList<>(); for (String custom : previousState.metadata().customs().keySet()) { @@ -88,7 +89,8 @@ public ClusterStateDiffManifest(String fromStateUUID, boolean coordinationMetadataUpdated, boolean settingsMetadataUpdated, boolean templatesMetadataUpdated, - Map customMetadataUpdated, List customMetadataDeleted, + List customMetadataUpdated, + List customMetadataDeleted, List indicesUpdated, List indicesDeleted, boolean clusterBlocksUpdated, @@ -132,10 +134,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); builder.endObject(); - // ToDo: add the custom metadata diff when we add a parser for this -// for (Map.Entry entry : customMetadataUpdated.entrySet()) { -// if (entry.getValue()) builder.field("customs_" + entry.getKey(), true); -// } + builder.startObject(METADATA_CUSTOM_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String custom : customMetadataUpdated) { + builder.value(custom); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String custom : customMetadataDeleted) { + builder.value(custom); + } + builder.endArray(); + builder.endObject(); } builder.endObject(); builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); @@ -206,6 +216,21 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); } } + } else if (currentFieldName.equals(METADATA_CUSTOM_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.customMetadataUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.customMetadataDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } } else { throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); } @@ -331,7 +356,7 @@ public boolean isTemplatesMetadataUpdated() { return templatesMetadataUpdated; } - public Map getCustomMetadataUpdated() { + public List getCustomMetadataUpdated() { return customMetadataUpdated; } @@ -373,7 +398,7 @@ public static class Builder { private boolean coordinationMetadataUpdated; private boolean settingsMetadataUpdated; private boolean templatesMetadataUpdated; - private Map customMetadataUpdated; + private List customMetadataUpdated; private List customMetadataDeleted; private List indicesUpdated; private List indicesDeleted; @@ -408,7 +433,7 @@ public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { return this; } - public Builder customMetadataUpdated(Map customMetadataUpdated) { + public Builder customMetadataUpdated(List customMetadataUpdated) { this.customMetadataUpdated = customMetadataUpdated; return this; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 416a543f659cd..4a06d7928e91b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -89,7 +89,6 @@ */ public class RemoteClusterStateService implements Closeable { public static final int RETAINED_MANIFESTS = 10; - public static final int SKIP_CLEANUP_STATE_CHANGES = 10; private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); @@ -1076,7 +1075,7 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata Map updatedCustomMetadata = new HashMap<>(); if (diff.getCustomMetadataUpdated() != null) { - for (String customType : diff.getCustomMetadataUpdated().keySet()) { + for (String customType : diff.getCustomMetadataUpdated()) { updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); } } @@ -1097,16 +1096,11 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata ); ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); + // remove the deleted indices from the metadata for (String index:diff.getIndicesDeleted()) { metadataBuilder.remove(index); } - if (diff.getCustomMetadataUpdated() != null) { - for (String customType : diff.getCustomMetadataUpdated().keySet()) { - Metadata.Custom custom = remoteGlobalMetadataManager.getCustomsMetadata(manifest.getClusterUUID(), - manifest.getCustomMetadataMap().get(customType).getUploadedFilename(), customType); - metadataBuilder.putCustom(customType, custom); - } - } + // remove the deleted metadata customs from the metadata if (diff.getCustomMetadataDeleted() != null) { for (String customType : diff.getCustomMetadataDeleted()) { metadataBuilder.removeCustom(customType); From a96304d39541bba98bfeda735961169b3f6c5e0c Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 23 May 2024 20:55:27 +0530 Subject: [PATCH 084/133] Add RemoteIndexMetadata unit tests Signed-off-by: Sooraj Sinha --- .../remote/ClusterMetadataManifest.java | 2 +- .../gateway/remote/RemoteCustomMetadata.java | 5 + .../gateway/remote/RemoteIndexMetadata.java | 6 - .../RemoteRoutingTableServiceTests.java | 2 - ...RemoteClusterStateCleanupManagerTests.java | 1 + .../RemoteClusterStateServiceTests.java | 4 +- .../remote/RemoteIndexMetadataTests.java | 154 ++++++++++++++++++ 7 files changed, 163 insertions(+), 11 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 41a400965dac4..aa179092949e3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -964,7 +964,7 @@ private static String componentPrefix(Object[] fields) { PARSER.declareString(ConstructingObjectParser.constructorArg(), COMPONENT_PREFIX_FIELD); } - static final String COMPONENT_PREFIX = "index--"; + public static final String COMPONENT_PREFIX = "index--"; private final String componentPrefix; private final String indexName; private final String indexUUID; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java index a98be23df571c..f70a99f2fa21c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java @@ -30,6 +30,11 @@ public class RemoteCustomMetadata extends AbstractRemoteBlobStoreObject public static final String CUSTOM_METADATA = "custom"; public static final String CUSTOM_DELIMITER = "--"; + public static final ChecksumBlobStoreFormat CUSTOM_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "custom", + METADATA_NAME_FORMAT, + Metadata.Custom::fromXContent + ); public final ChecksumBlobStoreFormat customBlobStoreFormat; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java index 6f031c3e020d3..3249927732988 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java @@ -108,10 +108,4 @@ public IndexMetadata deserialize(InputStream inputStream) throws IOException { return INDEX_METADATA_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); } - @Override - public String toString() { - return blobName + clusterUUID; - } - - } diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index e5dff93b6074d..162cf9b70cff8 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -56,7 +56,6 @@ public void setup() { remoteRoutingTableService = new RemoteRoutingTableService( repositoriesServiceSupplier, settings, - clusterSettings, new ThreadPool(settings) ); } @@ -75,7 +74,6 @@ public void testFailInitializationWhenRemoteRoutingDisabled() { () -> new RemoteRoutingTableService( repositoriesServiceSupplier, settings, - new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new ThreadPool(settings) ) ); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index d69864a9d2268..92eb573a866f4 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -69,6 +69,7 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 04a9efe9ae4ea..7a823caef8dae 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -1648,9 +1648,9 @@ private void mockBlobContainerForGlobalMetadata( for (Map.Entry entry : customFileMap.entrySet()) { String custom = entry.getKey(); String fileName = entry.getValue(); - when(blobContainer.readBlob(RemoteGlobalMetadataManager.CUSTOM_METADATA_FORMAT.blobName(fileName))).thenAnswer( + when(blobContainer.readBlob(RemoteCustomMetadata.CUSTOM_METADATA_FORMAT.blobName(fileName))).thenAnswer( (invocation) -> { - BytesReference bytesReference = RemoteGlobalMetadataManager.CUSTOM_METADATA_FORMAT.serialize( + BytesReference bytesReference = RemoteCustomMetadata.CUSTOM_METADATA_FORMAT.serialize( metadata.custom(custom), fileName, blobStoreRepository.getCompressor(), diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java new file mode 100644 index 0000000000000..b049e381f0705 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java @@ -0,0 +1,154 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata.COMPONENT_PREFIX; +import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.Index; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteIndexMetadataTests extends OpenSearchTestCase { + + private static final String TEST_BLOB_NAME = "test-blob-name"; + private static final long VERSION = 5L; + + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testGet() { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + assertThat(remoteObjectForUpload.get(), is(indexMetadata)); + + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + assertThat(remoteObjectForDownload.get(), nullValue()); + } + + public void testClusterUUID() { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + // todo test after uploading as well + + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobPathParameters() { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(INDEX_PATH_TOKEN))); + assertThat(params.getFilePrefix(),is("metadata")); + } + + public void testGenerateBlobFileName() { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertThat(nameTokens[0] ,is("metadata")); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]) , is(VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]) , lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3] , is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION))); + } + + public void testGetUploadedMetadata() throws IOException { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertThat(uploadedMetadata.getComponent() ,is(COMPONENT_PREFIX+"test-index")); + assertThat(uploadedMetadata.getUploadedFilename() ,is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + assertThat(inputStream.available(), greaterThan(0)); + IndexMetadata readIndexMetadata = remoteObjectForUpload.deserialize(inputStream); + assertThat(readIndexMetadata, is(indexMetadata)); + } + } + + private IndexMetadata getIndexMetadata() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); + return new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .version(VERSION) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + } +} From 6d1990c43467f9ceab27ea87640325f3b2ca2419 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Fri, 24 May 2024 12:33:30 +0530 Subject: [PATCH 085/133] Enable remote routing by default with remote state Signed-off-by: Arpit Bandejiya --- .../cluster/routing/remote/RemoteRoutingTableService.java | 4 ++-- .../main/java/org/opensearch/common/util/FeatureFlags.java | 2 +- .../opensearch/node/remotestore/RemoteStoreNodeAttribute.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 170fe44554d27..266709894c1b7 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -73,7 +73,7 @@ public class RemoteRoutingTableService implements Closeable { */ public static final Setting REMOTE_ROUTING_TABLE_ENABLED_SETTING = Setting.boolSetting( "cluster.remote_store.routing.enabled", - false, + true, Setting.Property.NodeScope, Setting.Property.Final ); @@ -276,7 +276,7 @@ public void close() throws IOException { public void start() { assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; final String remoteStoreRepo = settings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY ); assert remoteStoreRepo != null : "Remote routing table repository is not configured"; final Repository repository = repositoriesService.get().repository(remoteStoreRepo); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 5acff85bbcf64..1da131ab2f56c 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -100,7 +100,7 @@ public class FeatureFlags { public static final Setting REMOTE_ROUTING_TABLE_EXPERIMENTAL_SETTING = Setting.boolSetting( REMOTE_ROUTING_TABLE_EXPERIMENTAL, - false, + true, Property.NodeScope ); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 29eaecf2024de..2943667feeacf 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -198,7 +198,7 @@ public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { } public static boolean isRemoteRoutingTableAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) .isEmpty() == false; } From e022d78af4a20f41a6f66f2a164a6887237c8f6e Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 24 May 2024 13:30:24 +0530 Subject: [PATCH 086/133] Correct the diff creation logic Signed-off-by: Sooraj Sinha --- .../remote/ClusterStateDiffManifest.java | 8 ++-- .../remote/RemoteIndexMetadataTests.java | 47 +++++++++++-------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 98dbadbdaefaa..3e65c005a724c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -61,12 +61,12 @@ public class ClusterStateDiffManifest implements ToXContentObject { ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { fromStateUUID = previousState.stateUUID(); toStateUUID = state.stateUUID(); - coordinationMetadataUpdated = Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); - settingsMetadataUpdated = Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); - templatesMetadataUpdated = Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); + coordinationMetadataUpdated = !Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); + settingsMetadataUpdated = !Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); + templatesMetadataUpdated = !Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); - clusterBlocksUpdated = state.blocks().equals(previousState.blocks()); + clusterBlocksUpdated = !state.blocks().equals(previousState.blocks()); discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); customMetadataUpdated = new ArrayList<>(); for (String custom : state.metadata().customs().keySet()) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java index b049e381f0705..2cae382458c83 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java @@ -10,7 +10,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; @@ -27,7 +26,6 @@ import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.compress.DeflateCompressor; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -72,66 +70,75 @@ public void tearDown() throws Exception { public void testGet() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); assertThat(remoteObjectForUpload.get(), is(indexMetadata)); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); assertThat(remoteObjectForDownload.get(), nullValue()); } public void testClusterUUID() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); } public void testFullBlobName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); - // todo test after uploading as well - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); } public void testBlobPathParameters() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertThat(params.getPathTokens(), is(List.of(INDEX_PATH_TOKEN))); - assertThat(params.getFilePrefix(),is("metadata")); + assertThat(params.getFilePrefix(), is("metadata")); } public void testGenerateBlobFileName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); - assertThat(nameTokens[0] ,is("metadata")); - assertThat(RemoteStoreUtils.invertLong(nameTokens[1]) , is(VERSION)); - assertThat(RemoteStoreUtils.invertLong(nameTokens[2]) , lessThanOrEqualTo(System.currentTimeMillis())); - assertThat(nameTokens[3] , is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION))); + assertThat(nameTokens[0], is("metadata")); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION))); } public void testGetUploadedMetadata() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); try (InputStream inputStream = remoteObjectForUpload.serialize()) { UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); - assertThat(uploadedMetadata.getComponent() ,is(COMPONENT_PREFIX+"test-index")); - assertThat(uploadedMetadata.getUploadedFilename() ,is(remoteObjectForUpload.getFullBlobName())); + assertThat(uploadedMetadata.getComponent(), is(COMPONENT_PREFIX + "test-index")); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); } } public void testSerDe() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, + clusterName, threadPool); try (InputStream inputStream = remoteObjectForUpload.serialize()) { assertThat(inputStream.available(), greaterThan(0)); IndexMetadata readIndexMetadata = remoteObjectForUpload.deserialize(inputStream); From ca2640b7a26698ae362f939084dcc8921a6bb59e Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Fri, 24 May 2024 10:21:41 +0530 Subject: [PATCH 087/133] Delete stale index routing files --- .../RemoteClusterStateCleanupManager.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 090dae144baf6..65798d539e43a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -24,6 +24,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import java.io.Closeable; import java.io.IOException; @@ -171,6 +172,7 @@ void deleteClusterMetadata( Set staleManifestPaths = new HashSet<>(); Set staleIndexMetadataPaths = new HashSet<>(); Set staleGlobalMetadataPaths = new HashSet<>(); + Set staleIndexRoutingPaths = new HashSet<>(); activeManifestBlobMetadata.forEach(blobMetadata -> { ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.getRemoteManifestManager().fetchRemoteClusterMetadataManifest( clusterName, @@ -187,6 +189,11 @@ void deleteClusterMetadata( filesToKeep.add(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); clusterMetadataManifest.getCustomMetadataMap().values().forEach(attribute -> filesToKeep.add(attribute.getUploadedFilename())); } + if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V3) { + clusterMetadataManifest.getIndicesRouting() + .forEach(uploadedIndicesRouting -> filesToKeep.add(uploadedIndicesRouting.getUploadedFilename())); + logger.info("[Active Manifest] Uploaded Indices Routing files to keep: {}", filesToKeep); + } }); staleManifestBlobMetadata.forEach(blobMetadata -> { ClusterMetadataManifest clusterMetadataManifest = remoteClusterStateService.getRemoteManifestManager().fetchRemoteClusterMetadataManifest( @@ -205,6 +212,18 @@ void deleteClusterMetadata( addStaleGlobalMetadataPath(attribute.getUploadedFilename(), filesToKeep, staleGlobalMetadataPaths); }); } + if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V3) { + clusterMetadataManifest.getIndicesRouting().forEach(uploadedIndicesRouting -> { + if (filesToKeep.contains(uploadedIndicesRouting.getUploadedFilename()) == false){ + staleIndexRoutingPaths.add( + new BlobPath().add(RemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN).add(uploadedIndicesRouting.getIndexUUID()).buildAsString() + + uploadedIndicesRouting.getUploadedFilename() + ); + logger.debug("Stale files path in stale manifest: {}", new BlobPath().add(RemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN).add(uploadedIndicesRouting.getIndexUUID()).buildAsString() + + uploadedIndicesRouting.getUploadedFilename()); + } + }); + } clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) { @@ -224,6 +243,7 @@ void deleteClusterMetadata( deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleGlobalMetadataPaths)); deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexMetadataPaths)); deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleManifestPaths)); + deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexRoutingPaths)); } catch (IllegalStateException e) { logger.error("Error while fetching Remote Cluster Metadata manifests", e); } catch (IOException e) { From 9aa707e1093a45ed27049927f38944bcc7cb4840 Mon Sep 17 00:00:00 2001 From: Shailendra Singh Date: Fri, 24 May 2024 17:03:39 +0530 Subject: [PATCH 088/133] Fix stale file blob path for indices routing --- .../RemoteClusterStateCleanupManager.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 65798d539e43a..bf5527213c4b9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -28,6 +28,7 @@ import java.io.Closeable; import java.io.IOException; +import java.sql.Blob; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -192,7 +193,6 @@ void deleteClusterMetadata( if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V3) { clusterMetadataManifest.getIndicesRouting() .forEach(uploadedIndicesRouting -> filesToKeep.add(uploadedIndicesRouting.getUploadedFilename())); - logger.info("[Active Manifest] Uploaded Indices Routing files to keep: {}", filesToKeep); } }); staleManifestBlobMetadata.forEach(blobMetadata -> { @@ -215,12 +215,8 @@ void deleteClusterMetadata( if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V3) { clusterMetadataManifest.getIndicesRouting().forEach(uploadedIndicesRouting -> { if (filesToKeep.contains(uploadedIndicesRouting.getUploadedFilename()) == false){ - staleIndexRoutingPaths.add( - new BlobPath().add(RemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN).add(uploadedIndicesRouting.getIndexUUID()).buildAsString() - + uploadedIndicesRouting.getUploadedFilename() - ); - logger.debug("Stale files path in stale manifest: {}", new BlobPath().add(RemoteRoutingTableService.INDEX_ROUTING_PATH_TOKEN).add(uploadedIndicesRouting.getIndexUUID()).buildAsString() - + uploadedIndicesRouting.getUploadedFilename()); + staleIndexRoutingPaths.add(new BlobPath().buildAsString() + uploadedIndicesRouting.getUploadedFilename()); + logger.debug("Indices routing paths in stale manifest: {}", uploadedIndicesRouting.getUploadedFilename()); } }); } @@ -243,7 +239,7 @@ void deleteClusterMetadata( deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleGlobalMetadataPaths)); deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexMetadataPaths)); deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleManifestPaths)); - deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexRoutingPaths)); + deleteStaleIndexRoutingPaths(new ArrayList<>(staleIndexRoutingPaths)); } catch (IllegalStateException e) { logger.error("Error while fetching Remote Cluster Metadata manifests", e); } catch (IOException e) { @@ -348,6 +344,14 @@ void deleteStalePaths(String clusterName, String clusterUUID, List stale ); } + void deleteStaleIndexRoutingPaths(List stalePaths) throws IOException { + logger.debug(String.format(Locale.ROOT, "Deleting stale index routing files from remote - %s", stalePaths)); + getBlobStoreTransferService().deleteBlobs( + BlobPath.cleanPath(), + stalePaths + ); + } + /** * Purges all remote cluster state against provided cluster UUIDs * @param clusterState current state of the cluster From 3aaa6a8b0cf78c165e9c0b721a2beb1e45aa502c Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sat, 25 May 2024 14:57:40 +0530 Subject: [PATCH 089/133] Use isRemoteStateNode condition Signed-off-by: Sooraj Sinha --- .../coordination/PublicationTransportHandler.java | 14 +++++++++----- .../org/opensearch/cluster/node/DiscoveryNode.java | 12 ++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 1f1f795139651..aa2faab5ffd01 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -31,6 +31,7 @@ package org.opensearch.cluster.coordination; +import java.util.Locale; import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -230,14 +231,15 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest request) throws IOException { final Optional manifestOptional = remoteClusterStateService.getClusterMetadataManifestByTermVersion(request.getClusterName(), request.getClusterUUID(), request.term, request.version); if (manifestOptional.isPresent() == false) { - // todo change exception - throw new IncompatibleClusterStateVersionException("No remote state for term version"); + throw new IllegalStateException( + String.format(Locale.ROOT, "Manifest is not present for term - %s version - %s", request.term, request.version) + ); } ClusterMetadataManifest manifest = manifestOptional.get(); boolean applyFullState = false; final ClusterState lastSeen = lastSeenClusterState.get(); if (lastSeen == null) { - logger.debug("Diff cannot be applied as there is not last cluster state"); + logger.debug("Diff cannot be applied as there is no last cluster state"); applyFullState = true; } else if (manifest.getDiffManifest() == null) { logger.debug("There is no diff in the manifest"); @@ -250,12 +252,14 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish if (applyFullState == true) { ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest, transportService.getLocalNode().getId()); logger.debug("Downloaded full cluster state [{}]", clusterState); + fullClusterStateReceivedCount.incrementAndGet(); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.set(clusterState); return response; } else { ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get(), transportService.getLocalNode().getId()); logger.debug("Downloaded full cluster state from diff [{}]", clusterState); + compatibleClusterStateDiffReceivedCount.incrementAndGet(); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.compareAndSet(lastSeen, clusterState); return response; @@ -392,7 +396,7 @@ public void onFailure(Exception e) { } else { responseActionListener = listener; } - if (sendRemoteState && destination.isRemoteStoreNode()) { + if (sendRemoteState && destination.isRemoteStateNode()) { sendRemoteClusterState(destination, publishRequest.getAcceptedState(), responseActionListener); } else if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination); @@ -471,7 +475,7 @@ public String executor() { }; transportService.sendRequest(destination, PUBLISH_REMOTE_STATE_ACTION_NAME, remotePublishRequest, stateRequestOptions, responseHandler); } catch (Exception e) { - logger.warn(() -> new ParameterizedMessage("error sending cluster state to {}", destination), e); + logger.warn(() -> new ParameterizedMessage("error sending remote cluster state to {}", destination), e); listener.onFailure(e); } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 644e5f3de9352..fb27fccda0737 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -65,6 +65,7 @@ import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_API; import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_PARAM; import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; /** @@ -482,6 +483,17 @@ public boolean isRemoteStoreNode() { return this.getAttributes().keySet().stream().anyMatch(key -> key.startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)); } + /** + * Returns whether the node is a remote cluster state enabled node. + * @return true if the node contains remote cluster state node attribute, false otherwise + */ + public boolean isRemoteStateNode() { + return this.getAttributes() + .keySet() + .stream() + .anyMatch(key -> (key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY))); + } + /** * Returns a set of all the roles that the node has. The roles are returned in sorted order by the role name. *

From 5d70c591a405c2540426dd3cf3f66ebccfe8588f Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sat, 25 May 2024 15:30:09 +0530 Subject: [PATCH 090/133] Add metadata version --- .../opensearch/gateway/remote/RemoteGlobalMetadataManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 96aa8dc8bdaf6..b20a426cdb528 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -173,8 +173,7 @@ Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetada builder.templates(templatesMetadata); builder.clusterUUID(clusterMetadataManifest.getClusterUUID()); builder.clusterUUIDCommitted(clusterMetadataManifest.isClusterUUIDCommitted()); - //todo add metadata version - //builder.version(clusterMetadataManifest.met) + builder.version(clusterMetadataManifest.getMetadataVersion()); clusterMetadataManifest.getCustomMetadataMap() .forEach( (key, value) -> builder.putCustom( From 6a4c721b557d47eaf107f16b28e00d5b2a670d51 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sat, 25 May 2024 19:04:23 +0530 Subject: [PATCH 091/133] Correct blob path for download --- .../gateway/remote/AbstractRemoteBlobStoreObject.java | 2 +- .../opensearch/gateway/remote/RemoteClusterStateService.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java index 542a7074528a6..c8b2856b4b36a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java @@ -92,7 +92,7 @@ public BlobPath getBlobPathForUpload() { public BlobPath getBlobPathForDownload() { String[] pathTokens = extractBlobPathTokens(getFullBlobName()); - BlobPath blobPath = blobStoreRepository.basePath(); + BlobPath blobPath = new BlobPath(); for (String token : pathTokens) { blobPath = blobPath.add(token); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 4a06d7928e91b..7dcb9904b2ef3 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -456,8 +457,8 @@ private UploadedMetadataResults writeMetadataInParallel( int totalUploadTasks = indexToUpload.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size(); CountDownLatch latch = new CountDownLatch(totalUploadTasks); - Map> uploadTasks = new HashMap<>(totalUploadTasks); - Map results = new HashMap<>(totalUploadTasks); + Map> uploadTasks = new ConcurrentHashMap<>(totalUploadTasks); + Map results = new ConcurrentHashMap<>(totalUploadTasks); List exceptionList = Collections.synchronizedList(new ArrayList<>(totalUploadTasks)); LatchedActionListener listener = new LatchedActionListener<>( From fda30314c58148bbf921c3caf0849b9a032c8b03 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Sat, 25 May 2024 19:57:31 +0530 Subject: [PATCH 092/133] Custom toXContent implementations Signed-off-by: Shivansh Arora --- .../cluster/SnapshotDeletionsInProgress.java | 63 ++++- .../cluster/SnapshotsInProgress.java | 243 ++++++++++++++++++ .../org/opensearch/repositories/IndexId.java | 24 ++ .../org/opensearch/snapshots/SnapshotId.java | 21 ++ 4 files changed, 339 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index 6ef755ee34ebb..fed4332e99361 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.UUIDs; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; @@ -41,7 +42,6 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.repositories.RepositoryOperation; import org.opensearch.snapshots.SnapshotId; @@ -77,7 +77,7 @@ private SnapshotDeletionsInProgress(List entries) { assert assertNoConcurrentDeletionsForSameRepository(entries); } - public static SnapshotDeletionsInProgress of(List entries) { + public static SnapshotDeletionsInProgress of(List entries) { if (entries.isEmpty()) { return EMPTY; } @@ -195,11 +195,21 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repository", entry.repository()); builder.startArray("snapshots"); for (SnapshotId snapshot : entry.snapshots) { - builder.value(snapshot.getName()); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.startObject(); + builder.field("name", snapshot.getName()); + builder.field("uuid", snapshot.getUUID()); + builder.endObject(); + } else { + builder.value(snapshot.getName()); + } } builder.endArray(); builder.humanReadableField("start_time_millis", "start_time", new TimeValue(entry.startTime)); builder.field("repository_state_id", entry.repositoryStateId); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field("state", entry.state().value); + } // else we don't serialize it } builder.endObject(); } @@ -208,18 +218,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) throws IOException { - if (parser.currentToken() == null) { - parser.nextToken(); - } ensureFieldName(parser, parser.currentToken(), TYPE); parser.nextToken(); ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); List entries = new ArrayList<>(); while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - String repository, repositoryStateId; - List snapshotNames = new ArrayList<>(); - TimeValue startTime; + String repository = null; + long repositoryStateId = -1; + byte stateValue = -1; + List snapshotIds = new ArrayList<>(); + TimeValue startTime = null; while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); final String fieldName = parser.currentName(); @@ -231,7 +240,22 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th case "snapshots": ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { - snapshotNames.add(parser.text()); + String name = null; + String uuid = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String currentFieldName = parser.currentName(); + parser.nextToken(); + if ("name".equals(currentFieldName)) { + name = parser.text(); + } else if ("uuid".equals(currentFieldName)) { + uuid = parser.text(); + } else { + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + snapshotIds.add(new SnapshotId(name, uuid)); } break; case "start_time_millis": @@ -241,13 +265,17 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th startTime = TimeValue.parseTimeValue(parser.text(), "start_time"); break; case "repository_state_id": - repositoryStateId = parser.text(); + repositoryStateId = parser.longValue(); + break; + case "state": + stateValue = (byte) parser.intValue(); break; default: throw new IllegalArgumentException("unknown field [" + fieldName + "]"); } } -// entries.add(new Entry()) + assert startTime != null; + entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue))); } return SnapshotDeletionsInProgress.of(entries); } @@ -440,6 +468,17 @@ public static State readFrom(StreamInput in) throws IOException { } } + public static State fromValue(byte value) { + switch (value) { + case 0: + return WAITING; + case 1: + return STARTED; + default: + throw new IllegalArgumentException("No snapshot delete state for value [" + value + "]"); + } + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeByte(value); diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index d658f38430dd9..700b25d3d1e54 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -34,6 +34,7 @@ import org.opensearch.Version; import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.unit.TimeValue; @@ -45,6 +46,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoryOperation; import org.opensearch.repositories.RepositoryShardId; @@ -53,6 +55,7 @@ import org.opensearch.snapshots.SnapshotId; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -63,6 +66,10 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; +import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; + /** * Meta data about snapshots that are currently executing * @@ -732,16 +739,232 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SHARD, shardId.getId()); builder.field(STATE, status.state()); builder.field(NODE, status.nodeId()); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field(INDEX_UUID, shardId.getIndex().getUUID()); + if (status.generation() != null) builder.field(GENERATION, status.generation()); + if (status.reason() != null) builder.field(REASON, status.reason()); + } } builder.endObject(); } } builder.endArray(); builder.array(DATA_STREAMS, dataStreams.toArray(new String[0])); + if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { + builder.field(VERSION, version); + if (failure != null) builder.field(FAILURE, failure); + if (source != null) { + builder.field(SOURCE, source); + } + builder.field(USER_METADATA, userMetadata); + builder.startArray(CLONES); + for (RepositoryShardId shardId : clones.keySet()) { + builder.startObject(); + builder.field(INDEX, shardId.index().getName()); + builder.field(INDEX_ID, shardId.index().getId()); + builder.field(SHARD, shardId.shardId()); + ShardSnapshotStatus status = clones.get(shardId); + if (status.nodeId() != null) builder.field(NODE, status.nodeId()); + builder.field(STATE, status.state()); + if (status.generation() != null) builder.field(GENERATION, status.generation()); + if (status.reason() != null) builder.field(REASON, status.reason()); + builder.endObject(); + } + builder.field(REMOTE_STORE_INDEX_SHALLOW_COPY, remoteStoreIndexShallowCopy); + } builder.endObject(); return builder; } + public static Entry fromXContent(XContentParser parser) throws IOException { + String repository = null; + String snapshotName = null; + String snapshotUUID = null; + boolean includeGlobalState = false; + boolean partial = false; + String failure = null; + Version version = null; + SnapshotId source = null; + Map metadata = null; + byte state = -1; + List indices = new ArrayList<>(); + long startTime = 0; + long repositoryStateId = -1L; + Map shards = new HashMap<>(); + List dataStreams = new ArrayList<>(); + Map clones = new HashMap<>(); + boolean remoteStoreIndexShallowCopy = false; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String currentFieldName = parser.currentName(); + parser.nextToken(); + + switch (currentFieldName) { + case REPOSITORY: + repository = parser.text(); + break; + case SNAPSHOT: + snapshotName = parser.text(); + break; + case UUID: + snapshotUUID = parser.text(); + break; + case INCLUDE_GLOBAL_STATE: + includeGlobalState = parser.booleanValue(); + break; + case PARTIAL: + partial = parser.booleanValue(); + break; + case STATE: + state = (byte) parser.intValue(); + break; + case INDICES: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + indices.add(IndexId.fromXContent(parser)); + } + break; + case START_TIME_MILLIS: + startTime = parser.longValue(); + break; + case START_TIME: + startTime = TimeValue.parseTimeValue(parser.text(), null, currentFieldName).millis(); + break; + case REPOSITORY_STATE_ID: + repositoryStateId = parser.longValue(); + break; + case SHARDS: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + String index = null; + String indexUUID = null; + int shardId = -1; + String nodeId = null; + ShardState shardState = null; + String reason = null; + String generation = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + final String currentShardField = parser.currentName(); + parser.nextToken(); + switch (currentShardField) { + case INDEX: + index = parser.text(); + break; + case SHARD: + shardId = parser.intValue(); + break; + case INDEX_UUID: + indexUUID = parser.text(); + break; + case NODE: + nodeId = parser.text(); + break; + case STATE: + shardState = ShardState.fromValue((byte) parser.intValue()); + break; + case REASON: + reason = parser.text(); + break; + case GENERATION: + generation = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentShardField + "]"); + } + } + shards.put(new ShardId(index, indexUUID, shardId), + reason != null ? new ShardSnapshotStatus(nodeId, shardState, reason, generation) : + new ShardSnapshotStatus(nodeId, shardState, generation)); + } + break; + case DATA_STREAMS: + dataStreams = parseStringList(parser); + break; + case FAILURE: + failure = parser.text(); + break; + case SOURCE: + source = SnapshotId.fromXContent(parser); + break; + case USER_METADATA: + metadata = parser.map(); + break; + case CLONES: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + String index = null; + String indexId = null; + int shardId = -1; + byte snapshotShardStatus = -1; + String nodeId = null; + String reason = null; + String generation = null; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case INDEX: + index = parser.text(); + break; + case INDEX_ID: + indexId = parser.text(); + break; + case SHARD: + shardId = parser.intValue(); + break; + case STATE: + snapshotShardStatus = (byte) parser.intValue(); + break; + case NODE: + nodeId = parser.text(); + break; + case REASON: + reason = parser.text(); + break; + case GENERATION: + generation = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + clones.put(new RepositoryShardId(new IndexId(index, indexId), shardId), + reason != null ? new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), reason, generation) : + new ShardSnapshotStatus(nodeId, ShardState.fromValue(snapshotShardStatus), generation)); + } + break; + case VERSION: + version = Version.fromString(parser.text()); + break; + case REMOTE_STORE_INDEX_SHALLOW_COPY: + remoteStoreIndexShallowCopy = parser.booleanValue(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + Snapshot snapshot = new Snapshot(repository, new SnapshotId(snapshotName, snapshotUUID)); + return new Entry( + snapshot, + includeGlobalState, + partial, + State.fromValue(state), + indices, + dataStreams, + startTime, + repositoryStateId, + shards, + failure, + metadata, + version, + source, + clones, + remoteStoreIndexShallowCopy + ); + } + @Override public void writeTo(StreamOutput out) throws IOException { snapshot.writeTo(out); @@ -1057,6 +1280,12 @@ public void writeTo(StreamOutput out) throws IOException { private static final String INCLUDE_GLOBAL_STATE = "include_global_state"; private static final String PARTIAL = "partial"; private static final String STATE = "state"; + private static final String VERSION = "version"; + private static final String FAILURE = "failure"; + private static final String SOURCE = "source"; + private static final String USER_METADATA = "user_metadata"; + private static final String CLONES = "clones"; + private static final String REMOTE_STORE_INDEX_SHALLOW_COPY = "remote_store_index_shallow_copy"; private static final String INDICES = "indices"; private static final String DATA_STREAMS = "data_streams"; private static final String START_TIME_MILLIS = "start_time_millis"; @@ -1064,6 +1293,10 @@ public void writeTo(StreamOutput out) throws IOException { private static final String REPOSITORY_STATE_ID = "repository_state_id"; private static final String SHARDS = "shards"; private static final String INDEX = "index"; + private static final String INDEX_ID = "index_id"; + private static final String INDEX_UUID = "index_uuid"; + private static final String GENERATION = "generation"; + private static final String REASON = "reason"; private static final String SHARD = "shard"; private static final String NODE = "node"; @@ -1077,6 +1310,16 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par return builder; } + public static SnapshotsInProgress fromXContent(XContentParser parser) throws IOException { + ensureFieldName(parser, parser.currentToken(), SNAPSHOTS); + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser); + List entries = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + entries.add(Entry.fromXContent(parser)); + } + return SnapshotsInProgress.of(entries); + } + /** * The shard state. * diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index 87a0063e8c21b..1a2e797c62847 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -36,6 +36,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; @@ -44,6 +45,8 @@ import java.io.IOException; import java.util.Objects; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * Represents a single snapshotted index in the repository. * @@ -133,4 +136,25 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.endObject(); return builder; } + + public static IndexId fromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + String name = null; + String id = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case NAME: + name = parser.text(); + break; + case ID: + id = parser.text(); + break; + default: + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + return new IndexId(name, id); + } } diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java index 4eeb956a0cb19..31b18f14c854b 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotId.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotId.java @@ -38,6 +38,8 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; import java.io.IOException; import java.util.Objects; @@ -145,4 +147,23 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + public static SnapshotId fromXContent(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + String name = null; + String uuid = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + final String currentFieldName = parser.currentName(); + parser.nextToken(); + if (NAME.equals(currentFieldName)) { + name = parser.text(); + } else if (UUID.equals(currentFieldName)) { + uuid = parser.text(); + } else { + throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); + } + } + return new SnapshotId(name, uuid); + } } From 1ea5b01c111a03c27839f7b86f15900239907cab Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Sun, 26 May 2024 03:21:10 +0530 Subject: [PATCH 093/133] Add indexMetadataListener count to latch Signed-off-by: Shivansh Arora --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 7dcb9904b2ef3..63fc6d3d398a5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -333,7 +333,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( ) == false; // ToDo: check if these needs to be updated or not final boolean updateDiscoveryNodes = clusterState.getNodes().delta(previousClusterState.getNodes()).hasChanges(); - final boolean updateClusterBlocks = clusterState.blocks().equals(previousClusterState.blocks()); + final boolean updateClusterBlocks = !clusterState.blocks().equals(previousClusterState.blocks()); // Write Index Metadata final Map previousStateIndexMetadataByName = new HashMap<>(); @@ -454,7 +454,7 @@ private UploadedMetadataResults writeMetadataInParallel( boolean uploadDiscoveryNodes, boolean uploadClusterBlock, List indicesRoutingToUpload) throws IOException { - int totalUploadTasks = indexToUpload.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata + int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size(); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new ConcurrentHashMap<>(totalUploadTasks); From 9d687fa2650f9cdb605cebf80850c446dfda354d Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sun, 26 May 2024 16:28:04 +0530 Subject: [PATCH 094/133] Change RemoteObject interface signature Signed-off-by: Sooraj Sinha --- .../gateway/remote/AbstractRemoteBlobStoreObject.java | 10 ++++++---- .../remote/RemoteClusterStateAttributesManager.java | 2 +- .../gateway/remote/RemoteGlobalMetadataManager.java | 2 +- .../gateway/remote/RemoteIndexMetadataManager.java | 2 +- .../org/opensearch/gateway/remote/RemoteObject.java | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java index c8b2856b4b36a..aa48b5c884e79 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java @@ -56,12 +56,14 @@ public String getBlobFileName() { public abstract UploadedMetadata getUploadedMetadata(); @Override - public CheckedRunnable writeAsync(ActionListener listener) { - return () -> { - assert get() != null; + public void writeAsync(ActionListener listener) { + assert get() != null; + try { InputStream inputStream = serialize(); transferService.uploadBlob(inputStream, getBlobPathForUpload(), getBlobFileName(), WritePriority.URGENT, listener); - }; + } catch (Exception e) { + listener.onFailure(e); + } } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 5c7eb4cc25fee..380fd5670558d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -56,7 +56,7 @@ CheckedRunnable getAsyncMetadataWriteAction( ), ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) ); - return remoteObject.writeAsync(completionListener); + return () -> remoteObject.writeAsync(completionListener); } private AbstractRemoteBlobStoreObject getRemoteObject(ToXContent componentData, long stateVersion, String clusterUUID) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index b20a426cdb528..c74114509884f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -116,7 +116,7 @@ CheckedRunnable getAsyncMetadataWriteAction( ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException("Upload failed", ex)) ); - return remoteBlobStoreObject.writeAsync(completionListener); + return () -> remoteBlobStoreObject.writeAsync(completionListener); } CheckedRunnable getAsyncMetadataReadAction( diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index fc90ba82999ae..42a1fac13653a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -68,7 +68,7 @@ CheckedRunnable getIndexMetadataAsyncAction(IndexMetadata indexMeta ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) ); - return remoteIndexMetadata.writeAsync(completionListener); + return () -> remoteIndexMetadata.writeAsync(completionListener); } CheckedRunnable getAsyncIndexMetadataReadAction( diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java index 8802ab0e96c5a..34af997206e84 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java @@ -20,7 +20,7 @@ public interface RemoteObject { public InputStream serialize() throws IOException; public T deserialize(InputStream inputStream) throws IOException; - public CheckedRunnable writeAsync(ActionListener listener); + public void writeAsync(ActionListener listener); public T read() throws IOException; public void readAsync(ActionListener listener); From 677b632a79b9da08dee8cadd4f71d8a662351553 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sun, 26 May 2024 23:55:25 +0530 Subject: [PATCH 095/133] refactor manifest manager Signed-off-by: Sooraj Sinha --- .../remote/RemoteClusterMetadataManifest.java | 154 +++++++++++++++++ .../RemoteClusterStateCleanupManager.java | 17 +- .../remote/RemoteClusterStateService.java | 2 +- .../remote/RemoteIndexMetadataManager.java | 2 +- .../gateway/remote/RemoteManifestManager.java | 159 ++++-------------- ...RemoteClusterStateCleanupManagerTests.java | 4 +- .../RemoteClusterStateServiceTests.java | 8 +- .../remote/RemoteManifestManagerTests.java | 74 ++------ 8 files changed, 207 insertions(+), 213 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterMetadataManifest.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterMetadataManifest.java new file mode 100644 index 0000000000000..54e160e961671 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterMetadataManifest.java @@ -0,0 +1,154 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.common.io.Streams; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteClusterMetadataManifest extends AbstractRemoteBlobStoreObject { + + public static final String MANIFEST_PATH_TOKEN = "manifest"; + public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; + + public static final String MANIFEST_FILE_PREFIX = "manifest"; + public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; + + /** + * Manifest format compatible with older codec v0, where codec version was missing. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V0 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); + /** + * Manifest format compatible with older codec v1, where global metadata was missing. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = + new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); + + /** + * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. + */ + public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( + "cluster-metadata-manifest", + METADATA_MANIFEST_NAME_FORMAT, + ClusterMetadataManifest::fromXContent + ); + + private ClusterMetadataManifest clusterMetadataManifest; + private String blobName; + private final String clusterUUID; + + public RemoteClusterMetadataManifest(ClusterMetadataManifest clusterMetadataManifest, String clusterUUID, BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.clusterMetadataManifest = clusterMetadataManifest; + this.clusterUUID = clusterUUID; + } + + public RemoteClusterMetadataManifest(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.blobName = blobName; + this.clusterUUID = clusterUUID; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(MANIFEST_PATH_TOKEN), MANIFEST_FILE_PREFIX); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ + // + String blobFileName = String.join( + DELIMITER, + MANIFEST_PATH_TOKEN, + RemoteStoreUtils.invertLong(clusterMetadataManifest.getClusterTerm()), + RemoteStoreUtils.invertLong(clusterMetadataManifest.getStateVersion()), + (clusterMetadataManifest.isCommitted() ? "C" : "P"), // C for committed and P for published + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(clusterMetadataManifest.getCodecVersion()) // Keep the codec version at last place only, during read we reads last place to + // determine codec version. + ); + // setting the full blob path with name for future access + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public UploadedMetadata getUploadedMetadata() { + return new UploadedMetadataAttribute(MANIFEST_PATH_TOKEN, blobName); + } + + @Override + public ClusterMetadataManifest get() { + return clusterMetadataManifest; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + @Override + public InputStream serialize() throws IOException { + return CLUSTER_METADATA_MANIFEST_FORMAT.serialize(clusterMetadataManifest, generateBlobFileName(), getCompressor(), + RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public ClusterMetadataManifest deserialize(InputStream inputStream) throws IOException { + ChecksumBlobStoreFormat blobStoreFormat = getClusterMetadataManifestBlobStoreFormat(); + return blobStoreFormat.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + } + + private int getManifestCodecVersion() { + assert blobName != null; + String[] splitName = blobName.split(DELIMITER); + if (splitName.length == SPLITTED_MANIFEST_FILE_LENGTH) { + return Integer.parseInt(splitName[splitName.length - 1]); // Last value would be codec version. + } else if (splitName.length < SPLITTED_MANIFEST_FILE_LENGTH) { // Where codec is not part of file name, i.e. default codec version 0 + // is used. + return ClusterMetadataManifest.CODEC_V0; + } else { + throw new IllegalArgumentException("Manifest file name is corrupted"); + } + } + + private ChecksumBlobStoreFormat getClusterMetadataManifestBlobStoreFormat() { + long codecVersion = getManifestCodecVersion(); + if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { + return CLUSTER_METADATA_MANIFEST_FORMAT; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V1; + } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { + return CLUSTER_METADATA_MANIFEST_FORMAT_V0; + } + throw new IllegalArgumentException("Cluster metadata manifest file is corrupted, don't have valid codec version"); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index bf5527213c4b9..3375d3b28cc2a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -36,12 +36,10 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_METADATA_FORMAT; import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; /** * A Manager which provides APIs to clean up stale cluster state files and runs an async stale cleanup task @@ -152,12 +150,7 @@ void cleanUpStaleFiles() { private void addStaleGlobalMetadataPath(String fileName, Set filesToKeep, Set staleGlobalMetadataPaths) { if (!filesToKeep.contains(fileName)) { - String[] splitPath = fileName.split("/"); - staleGlobalMetadataPaths.add( - new BlobPath().add(GLOBAL_METADATA_PATH_TOKEN).buildAsString() + GLOBAL_METADATA_FORMAT.blobName( - splitPath[splitPath.length - 1] - ) - ); + staleGlobalMetadataPaths.add(fileName); } } @@ -181,9 +174,9 @@ void deleteClusterMetadata( blobMetadata.name() ); clusterMetadataManifest.getIndices() - .forEach(uploadedIndexMetadata -> filesToKeep.add(uploadedIndexMetadata.getUploadedFilename())); + .forEach(uploadedIndexMetadata -> filesToKeep.add(RemoteClusterStateUtils.getFormattedFileName(uploadedIndexMetadata.getUploadedFilename(), clusterMetadataManifest.getCodecVersion()))); if (clusterMetadataManifest.getCodecVersion() == ClusterMetadataManifest.CODEC_V1) { - filesToKeep.add(clusterMetadataManifest.getGlobalMetadataFileName()); + filesToKeep.add(RemoteClusterStateUtils.getFormattedFileName(clusterMetadataManifest.getGlobalMetadataFileName(), clusterMetadataManifest.getCodecVersion())); } else if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V2) { filesToKeep.add(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); filesToKeep.add(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 63fc6d3d398a5..0b7f9db3cb667 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -771,7 +771,7 @@ public void start() { remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings,threadpool, getBlobStoreTransferService(), clusterName); remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings,threadpool, clusterName, getBlobStoreTransferService()); remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(getBlobStoreTransferService(), blobStoreRepository, threadpool, clusterName); - remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, nodeId); + remoteManifestManager = new RemoteManifestManager(getBlobStoreTransferService(), blobStoreRepository, clusterSettings, nodeId, threadpool); remoteClusterStateCleanupManager.start(); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index 42a1fac13653a..5f5459994e51e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -66,7 +66,7 @@ CheckedRunnable getIndexMetadataAsyncAction(IndexMetadata indexMeta resp -> latchedActionListener.onResponse( remoteIndexMetadata.getUploadedMetadata() ), - ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().toString(), ex)) + ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().getName(), ex)) ); return () -> remoteIndexMetadata.writeAsync(completionListener); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index f597e43149a07..e9b7a1c1f0bb9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -21,6 +21,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; @@ -34,6 +35,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.opensearch.threadpool.ThreadPool; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; @@ -41,11 +43,6 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; public class RemoteManifestManager { - public static final String MANIFEST_PATH_TOKEN = "manifest"; - public static final String MANIFEST_FILE_PREFIX = "manifest"; - public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; - public static final int SPLITED_MANIFEST_FILE_LENGTH = 6; // file name manifest__term__version__C/P__timestamp__codecversion public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); @@ -56,71 +53,22 @@ public class RemoteManifestManager { Setting.Property.NodeScope ); - /** - * Manifest format compatible with older codec v0, where codec version was missing. - */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V0 = - new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV0); - /** - * Manifest format compatible with older codec v1, where global metadata was missing. - */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V1 = - new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV1); - - /** - * Manifest format compatible with codec v2, where we introduced codec versions/global metadata. - */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( - "cluster-metadata-manifest", - METADATA_MANIFEST_NAME_FORMAT, - ClusterMetadataManifest::fromXContent - ); - + private final BlobStoreTransferService blobStoreTransferService; private final BlobStoreRepository blobStoreRepository; private volatile TimeValue metadataManifestUploadTimeout; private final String nodeId; + private final ThreadPool threadPool; private static final Logger logger = LogManager.getLogger(RemoteManifestManager.class); - RemoteManifestManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, String nodeId) { + RemoteManifestManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, String nodeId, ThreadPool threadPool) { + this.blobStoreTransferService = blobStoreTransferService; this.blobStoreRepository = blobStoreRepository; this.metadataManifestUploadTimeout = clusterSettings.get(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING); this.nodeId = nodeId; + this.threadPool = threadPool; clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); } - private ClusterMetadataManifest uploadV1Manifest( - ClusterState clusterState, - List uploadedIndexMetadata, - String previousClusterUUID, - String globalMetadataFileName, - boolean committed - ) throws IOException { - synchronized (this) { - final String manifestFileName = getManifestFileName( - clusterState.term(), - clusterState.version(), - committed, - ClusterMetadataManifest.CODEC_V1 - ); - ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() - .clusterTerm(clusterState.term()) - .stateVersion(clusterState.getVersion()) - .clusterUUID(clusterState.metadata().clusterUUID()) - .stateUUID(clusterState.stateUUID()) - .opensearchVersion(Version.CURRENT) - .nodeId(nodeId) - .committed(committed) - .codecVersion(ClusterMetadataManifest.CODEC_V1) - .globalMetadataFileName(globalMetadataFileName) - .indices(uploadedIndexMetadata) - .previousClusterUUID(previousClusterUUID) - .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) - .build(); - writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); - return manifest; - } - } - ClusterMetadataManifest uploadManifest( ClusterState clusterState, List uploadedIndexMetadata, @@ -133,14 +81,8 @@ ClusterMetadataManifest uploadManifest( ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocksMetadata, ClusterStateDiffManifest clusterDiffManifest, List routingIndexMetadata, boolean committed - ) throws IOException { + ) { synchronized (this) { - final String manifestFileName = getManifestFileName( - clusterState.term(), - clusterState.version(), - committed, - MANIFEST_CURRENT_CODEC_VERSION - ); ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder(); manifestBuilder.clusterTerm(clusterState.term()) .stateVersion(clusterState.getVersion()) @@ -149,7 +91,7 @@ ClusterMetadataManifest uploadManifest( .opensearchVersion(Version.CURRENT) .nodeId(nodeId) .committed(committed) - .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .codecVersion(RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION) .indices(uploadedIndexMetadata) .previousClusterUUID(previousClusterUUID) .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) @@ -164,18 +106,15 @@ ClusterMetadataManifest uploadManifest( .indicesRouting(routingIndexMetadata) .metadataVersion(clusterState.metadata().version()); final ClusterMetadataManifest manifest = manifestBuilder.build(); - writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); + writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest); return manifest; } } - private void writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest, String fileName) - throws IOException { + private void writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest) { AtomicReference result = new AtomicReference(); AtomicReference exceptionReference = new AtomicReference(); - final BlobContainer metadataManifestContainer = manifestContainer(clusterName, clusterUUID); - // latch to wait until upload is not finished CountDownLatch latch = new CountDownLatch(1); @@ -183,14 +122,8 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); }, ex -> { exceptionReference.set(ex); }), latch); - getClusterMetadataManifestBlobStoreFormat(fileName).writeAsyncWithUrgentPriority( - uploadManifest, - metadataManifestContainer, - fileName, - blobStoreRepository.getCompressor(), - completionListener, - FORMAT_PARAMS - ); + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(uploadManifest, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + remoteClusterMetadataManifest.writeAsync(completionListener); try { if (latch.await(getMetadataManifestUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { @@ -212,7 +145,7 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust } logger.debug( "Metadata manifest file [{}] written during [{}] phase. ", - fileName, + remoteClusterMetadataManifest.getBlobFileName(), uploadManifest.isCommitted() ? "commit" : "publish" ); } @@ -229,6 +162,14 @@ public Optional getLatestClusterMetadataManifest(String return latestManifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); } + /** + * Fetch the cluster metadata manifest using term and version + * @param clusterName uuid of cluster state to refer to in remote + * @param clusterUUID name of the cluster + * @param term election term of the cluster + * @param version cluster state version number + * @return ClusterMetadataManifest + */ public Optional getClusterMetadataManifestByTermVersion(String clusterName, String clusterUUID, long term, long version) { Optional manifestFileName = getManifestFileNameByTermVersion(clusterName, clusterUUID, term, version); return manifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); @@ -244,11 +185,8 @@ public Optional getClusterMetadataManifestByTermVersion ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, String clusterUUID, String filename) throws IllegalStateException { try { - return getClusterMetadataManifestBlobStoreFormat(filename).read( - manifestContainer(clusterName, clusterUUID), - filename, - blobStoreRepository.getNamedXContentRegistry() - ); + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(filename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + return remoteClusterMetadataManifest.read(); } catch (IOException e) { throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); } @@ -270,38 +208,13 @@ Map getLatestManifestForAllClusterUUIDs(String return manifestsByClusterUUID; } - private ChecksumBlobStoreFormat getClusterMetadataManifestBlobStoreFormat(String fileName) { - long codecVersion = getManifestCodecVersion(fileName); - if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { - return CLUSTER_METADATA_MANIFEST_FORMAT; - } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { - return CLUSTER_METADATA_MANIFEST_FORMAT_V1; - } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { - return CLUSTER_METADATA_MANIFEST_FORMAT_V0; - } - - throw new IllegalArgumentException("Cluster metadata manifest file is corrupted, don't have valid codec version"); - } - - private int getManifestCodecVersion(String fileName) { - String[] splitName = fileName.split(DELIMITER); - if (splitName.length == SPLITED_MANIFEST_FILE_LENGTH) { - return Integer.parseInt(splitName[splitName.length - 1]); // Last value would be codec version. - } else if (splitName.length < SPLITED_MANIFEST_FILE_LENGTH) { // Where codec is not part of file name, i.e. default codec version 0 - // is used. - return ClusterMetadataManifest.CODEC_V0; - } else { - throw new IllegalArgumentException("Manifest file name is corrupted"); - } - } - private BlobContainer manifestContainer(String clusterName, String clusterUUID) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest return blobStoreRepository.blobStore().blobContainer(getManifestFolderPath(clusterName, clusterUUID)); } BlobPath getManifestFolderPath(String clusterName, String clusterUUID) { - return getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(MANIFEST_PATH_TOKEN); + return getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN); } public TimeValue getMetadataManifestUploadTimeout() { @@ -338,24 +251,10 @@ private List getManifestFileNames(String clusterName, String clust } } - static String getManifestFileName(long term, long version, boolean committed, int codecVersion) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ - return String.join( - DELIMITER, - MANIFEST_PATH_TOKEN, - RemoteStoreUtils.invertLong(term), - RemoteStoreUtils.invertLong(version), - (committed ? "C" : "P"), // C for committed and P for published - RemoteStoreUtils.invertLong(System.currentTimeMillis()), - String.valueOf(codecVersion) // Keep the codec version at last place only, during read we reads last place to - // determine codec version. - ); - } - static String getManifestFilePrefixForTermVersion(long term, long version) { return String.join( DELIMITER, - MANIFEST_FILE_PREFIX, + RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX, RemoteStoreUtils.invertLong(term), RemoteStoreUtils.invertLong(version) ) + DELIMITER; @@ -369,9 +268,9 @@ static String getManifestFilePrefixForTermVersion(long term, long version) { * @return latest ClusterMetadataManifest filename */ private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { - List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, MANIFEST_FILE_PREFIX + DELIMITER, 1); + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX + DELIMITER, 1); if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { - return Optional.of(manifestFilesMetadata.get(0).name()); + return Optional.of(getManifestFolderPath(clusterName, clusterUUID).buildAsString() + manifestFilesMetadata.get(0).name()); } logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); return Optional.empty(); @@ -381,7 +280,7 @@ private Optional getManifestFileNameByTermVersion(String clusterName, St final String filePrefix = getManifestFilePrefixForTermVersion(term, version); List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, filePrefix, 1); if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { - return Optional.of(manifestFilesMetadata.get(0).name()); + return Optional.of(getManifestFolderPath(clusterName, clusterUUID).buildAsString() + manifestFilesMetadata.get(0).name()); } logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}, term: {}, version: {}", clusterName, clusterUUID, term, version); return Optional.empty(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 92eb573a866f4..0918236638016 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -70,8 +70,8 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 7a823caef8dae..a2fa691a16e4e 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -89,15 +89,13 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.common.util.FeatureFlags.REMOTE_ROUTING_TABLE_EXPERIMENTAL; -import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_CURRENT_CODEC_VERSION; -import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; +import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -1553,7 +1551,7 @@ private void mockBlobContainer( when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) .thenReturn(Arrays.asList(blobMetadata)); - BytesReference bytes = RemoteManifestManager.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( + BytesReference bytes = RemoteClusterMetadataManifest.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( clusterMetadataManifest, manifestFileName, blobStoreRepository.getCompressor(), @@ -1594,7 +1592,7 @@ private void mockBlobContainerForGlobalMetadata( when(blobContainer.listBlobsByPrefixInSortedOrder("manifest" + DELIMITER, 1, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC)) .thenReturn(Arrays.asList(blobMetadata)); - BytesReference bytes = RemoteManifestManager.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( + BytesReference bytes = RemoteClusterMetadataManifest.CLUSTER_METADATA_MANIFEST_FORMAT.serialize( clusterMetadataManifest, mockManifestFileName, blobStoreRepository.getCompressor(), diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java index 10a718f6a627e..e0af7808a5853 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java @@ -8,38 +8,28 @@ package org.opensearch.gateway.remote; -import org.opensearch.Version; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; + +import java.io.IOException; +import org.junit.After; +import org.junit.Before; import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.core.index.Index; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; import org.opensearch.threadpool.TestThreadPool; -import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; -import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteManifestManager.MANIFEST_FILE_PREFIX; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class RemoteManifestManagerTests extends OpenSearchTestCase { private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; @@ -51,7 +41,7 @@ public class RemoteManifestManagerTests extends OpenSearchTestCase { public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); - remoteManifestManager = new RemoteManifestManager(blobStoreRepository, clusterSettings, "test-node-id"); + remoteManifestManager = new RemoteManifestManager(blobStoreTransferService, blobStoreRepository, clusterSettings, "test-node-id", new TestThreadPool("test")); blobStoreTransferService = mock(BlobStoreTransferService.class); blobStore = mock(BlobStore.class); when(blobStoreRepository.blobStore()).thenReturn(blobStore); @@ -78,46 +68,6 @@ public void testMetadataManifestUploadWaitTimeSetting() { assertEquals(metadataManifestUploadTimeout, remoteManifestManager.getMetadataManifestUploadTimeout().seconds()); } - public void testFileNames() { - final Index index = new Index("test-index", "index-uuid"); - final Settings idxSettings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) - .build(); - final IndexMetadata indexMetadata = new IndexMetadata.Builder(index.getName()).settings(idxSettings) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); - - String indexMetadataFileName = new RemoteIndexMetadata(indexMetadata, "cluster-uuid", blobStoreTransferService, blobStoreRepository, "cluster-name", new TestThreadPool("test")).generateBlobFileName(); - String[] splittedIndexMetadataFileName = indexMetadataFileName.split(DELIMITER); - assertEquals(4, indexMetadataFileName.split(DELIMITER).length); - assertEquals(METADATA_FILE_PREFIX, splittedIndexMetadataFileName[0]); - assertEquals(RemoteStoreUtils.invertLong(indexMetadata.getVersion()), splittedIndexMetadataFileName[1]); - assertEquals(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION), splittedIndexMetadataFileName[3]); - - verifyManifestFileNameWithCodec(MANIFEST_CURRENT_CODEC_VERSION); - verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V1); - verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V0); - } - - private void verifyManifestFileNameWithCodec(int codecVersion) { - int term = randomIntBetween(5, 10); - int version = randomIntBetween(5, 10); - String manifestFileName = RemoteManifestManager.getManifestFileName(term, version, true, codecVersion); - assertEquals(6, manifestFileName.split(DELIMITER).length); - String[] splittedName = manifestFileName.split(DELIMITER); - assertEquals(MANIFEST_FILE_PREFIX, splittedName[0]); - assertEquals(RemoteStoreUtils.invertLong(term), splittedName[1]); - assertEquals(RemoteStoreUtils.invertLong(version), splittedName[2]); - assertEquals("C", splittedName[3]); - assertEquals(String.valueOf(codecVersion), splittedName[5]); - - manifestFileName = RemoteManifestManager.getManifestFileName(term, version, false, codecVersion); - splittedName = manifestFileName.split(DELIMITER); - assertEquals("P", splittedName[3]); - } - public void testReadLatestMetadataManifestFailedIOException() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); From e12131dc15604101e2a7d68a0a3bb266097092db Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Mon, 27 May 2024 01:08:15 +0530 Subject: [PATCH 096/133] Refactor cleanup manager Signed-off-by: Sooraj Sinha --- .../RemoteClusterStateCleanupManager.java | 19 +++++++++---------- .../gateway/remote/RemoteIndexMetadata.java | 2 +- .../gateway/remote/RemoteManifestManager.java | 7 ++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 3375d3b28cc2a..d0ab53d337207 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -50,7 +50,7 @@ public class RemoteClusterStateCleanupManager implements Closeable { public static final int RETAINED_MANIFESTS = 10; public static final int SKIP_CLEANUP_STATE_CHANGES = 10; - public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT = TimeValue.timeValueMinutes(5); + public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_DEFAULT = TimeValue.timeValueSeconds(15); public static final TimeValue CLUSTER_STATE_CLEANUP_INTERVAL_MINIMUM = TimeValue.MINUS_ONE; /** @@ -194,9 +194,9 @@ void deleteClusterMetadata( clusterUUID, blobMetadata.name() ); - staleManifestPaths.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blobMetadata.name()); + staleManifestPaths.add(remoteClusterStateService.getRemoteManifestManager().getManifestFolderPath(clusterName, clusterUUID).buildAsString() + blobMetadata.name()); if (clusterMetadataManifest.getCodecVersion() == ClusterMetadataManifest.CODEC_V1) { - addStaleGlobalMetadataPath(clusterMetadataManifest.getGlobalMetadataFileName(), filesToKeep, staleGlobalMetadataPaths); + addStaleGlobalMetadataPath(RemoteClusterStateUtils.getFormattedFileName(clusterMetadataManifest.getGlobalMetadataFileName(), clusterMetadataManifest.getCodecVersion()), filesToKeep, staleGlobalMetadataPaths); } else if (clusterMetadataManifest.getCodecVersion() >= ClusterMetadataManifest.CODEC_V2) { addStaleGlobalMetadataPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename(), filesToKeep, staleGlobalMetadataPaths); addStaleGlobalMetadataPath(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename(), filesToKeep, staleGlobalMetadataPaths); @@ -217,8 +217,7 @@ void deleteClusterMetadata( clusterMetadataManifest.getIndices().forEach(uploadedIndexMetadata -> { if (filesToKeep.contains(uploadedIndexMetadata.getUploadedFilename()) == false) { staleIndexMetadataPaths.add( - new BlobPath().add(INDEX_PATH_TOKEN).add(uploadedIndexMetadata.getIndexUUID()).buildAsString() - + INDEX_METADATA_FORMAT.blobName(uploadedIndexMetadata.getUploadedFilename()) + RemoteClusterStateUtils.getFormattedFileName(uploadedIndexMetadata.getUploadedFilename(), clusterMetadataManifest.getCodecVersion()) ); } }); @@ -229,9 +228,9 @@ void deleteClusterMetadata( return; } - deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleGlobalMetadataPaths)); - deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleIndexMetadataPaths)); - deleteStalePaths(clusterName, clusterUUID, new ArrayList<>(staleManifestPaths)); + deleteStalePaths(new ArrayList<>(staleGlobalMetadataPaths)); + deleteStalePaths(new ArrayList<>(staleIndexMetadataPaths)); + deleteStalePaths(new ArrayList<>(staleManifestPaths)); deleteStaleIndexRoutingPaths(new ArrayList<>(staleIndexRoutingPaths)); } catch (IllegalStateException e) { logger.error("Error while fetching Remote Cluster Metadata manifests", e); @@ -329,10 +328,10 @@ public void onFailure(Exception e) { } // package private for testing - void deleteStalePaths(String clusterName, String clusterUUID, List stalePaths) throws IOException { + void deleteStalePaths(List stalePaths) throws IOException { logger.debug(String.format(Locale.ROOT, "Deleting stale files from remote - %s", stalePaths)); getBlobStoreTransferService().deleteBlobs( - RemoteClusterStateUtils.getCusterMetadataBasePath(remoteClusterStateService.getBlobStoreRepository(), clusterName, clusterUUID), + BlobPath.cleanPath(), stalePaths ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java index 3249927732988..b177747c8a249 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java @@ -72,7 +72,7 @@ public String getFullBlobName() { @Override public BlobPathParameters getBlobPathParameters() { - return new BlobPathParameters(List.of(INDEX_PATH_TOKEN), "metadata"); + return new BlobPathParameters(List.of(INDEX_PATH_TOKEN, indexMetadata.getIndexUUID()), "metadata"); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index e9b7a1c1f0bb9..c6bc8a64293d1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -185,7 +185,8 @@ public Optional getClusterMetadataManifestByTermVersion ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, String clusterUUID, String filename) throws IllegalStateException { try { - RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(filename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + String fullBlobName = getManifestFolderPath(clusterName, clusterUUID).buildAsString() + filename; + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(fullBlobName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); return remoteClusterMetadataManifest.read(); } catch (IOException e) { throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); @@ -270,7 +271,7 @@ static String getManifestFilePrefixForTermVersion(long term, long version) { private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX + DELIMITER, 1); if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { - return Optional.of(getManifestFolderPath(clusterName, clusterUUID).buildAsString() + manifestFilesMetadata.get(0).name()); + return Optional.of(manifestFilesMetadata.get(0).name()); } logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}", clusterName, clusterUUID); return Optional.empty(); @@ -280,7 +281,7 @@ private Optional getManifestFileNameByTermVersion(String clusterName, St final String filePrefix = getManifestFilePrefixForTermVersion(term, version); List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, filePrefix, 1); if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { - return Optional.of(getManifestFolderPath(clusterName, clusterUUID).buildAsString() + manifestFilesMetadata.get(0).name()); + return Optional.of(manifestFilesMetadata.get(0).name()); } logger.info("No manifest file present in remote store for cluster name: {}, cluster UUID: {}, term: {}, version: {}", clusterName, clusterUUID, term, version); return Optional.empty(); From f4f773f87cbdd9611301080f44e70b83879c1bfc Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Mon, 27 May 2024 14:44:38 +0530 Subject: [PATCH 097/133] Prevent loading ephemeral objects in restore flow Signed-off-by: Sooraj Sinha --- .../remote/RemoteClusterStateServiceIT.java | 3 +- .../PublicationTransportHandler.java | 2 +- .../remote/AbstractRemoteBlobStoreObject.java | 2 +- .../remote/RemoteClusterStateService.java | 12 +++---- .../recovery/RemoteStoreRestoreService.java | 2 +- .../blobstore/ChecksumBlobStoreFormat.java | 36 ++++--------------- .../RemoteClusterStateServiceTests.java | 14 +++++--- 7 files changed, 27 insertions(+), 44 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index 91bba40375b49..283d8bb3913f0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -160,7 +160,8 @@ public void testRemoteCleanupOnlyAfter10Updates() throws Exception { Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( cluster().getClusterName(), - getClusterState().metadata().clusterUUID() + getClusterState().metadata().clusterUUID(), + false ).getMetadata().getIndices(); assertEquals(replicaCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index aa2faab5ffd01..d553324bbf49d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -250,7 +250,7 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish } if (applyFullState == true) { - ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest, transportService.getLocalNode().getId()); + ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest, transportService.getLocalNode().getId(), true); logger.debug("Downloaded full cluster state [{}]", clusterState); fullClusterStateReceivedCount.incrementAndGet(); final PublishWithJoinResponse response = acceptState(clusterState); diff --git a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java index aa48b5c884e79..b826e8999f877 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java @@ -48,7 +48,7 @@ public String getBlobFileName() { generateBlobFileName(); } String[] pathTokens = getFullBlobName().split(PATH_DELIMITER); - return getFullBlobName().split(PATH_DELIMITER)[pathTokens.length - 1]; + return pathTokens[pathTokens.length - 1]; } public abstract String generateBlobFileName(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 0b7f9db3cb667..09dc6da3287c7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -803,7 +803,7 @@ BlobStore getBlobStore() { * @param clusterName name of the cluster * @return {@link IndexMetadata} */ - public ClusterState getLatestClusterState(String clusterName, String clusterUUID) throws IOException { + public ClusterState getLatestClusterState(String clusterName, String clusterUUID, boolean includeEphemeral) throws IOException { start(); Optional clusterMetadataManifest = remoteManifestManager.getLatestClusterMetadataManifest( clusterName, @@ -815,7 +815,7 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID ); } - return getClusterStateForManifest(clusterName, clusterMetadataManifest.get(), nodeId); + return getClusterStateForManifest(clusterName, clusterMetadataManifest.get(), nodeId, includeEphemeral); } private ClusterState readClusterStateInParallel( @@ -1041,7 +1041,7 @@ private ClusterState readClusterStateInParallel( .build(); } - public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId) throws IOException { + public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId, boolean includeEphemeral) throws IOException { return readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), manifest, @@ -1053,9 +1053,9 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada manifest.getCoordinationMetadata() != null, manifest.getSettingsMetadata() != null, manifest.getTemplatesMetadata() != null, - manifest.getDiscoveryNodesMetadata() != null, - manifest.getClusterBlocksMetadata() != null, - manifest.getIndicesRouting() + includeEphemeral && manifest.getDiscoveryNodesMetadata() != null, + includeEphemeral && manifest.getClusterBlocksMetadata() != null, + includeEphemeral ? manifest.getIndicesRouting() : Collections.emptyList() ); } diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java index fe90f24b0f544..e441e150e2acd 100644 --- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java +++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java @@ -152,7 +152,7 @@ public RemoteRestoreResult restore( throw new IllegalArgumentException("clusterUUID to restore from should be different from current cluster UUID"); } logger.info("Restoring cluster state from remote store from cluster UUID : [{}]", restoreClusterUUID); - remoteState = remoteClusterStateService.getLatestClusterState(currentState.getClusterName().value(), restoreClusterUUID); + remoteState = remoteClusterStateService.getLatestClusterState(currentState.getClusterName().value(), restoreClusterUUID, false); remoteState.getMetadata().getIndices().values().forEach(indexMetadata -> { indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata)); }); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java index 78b8fbeb45852..53b9b3300ffe3 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java @@ -31,7 +31,13 @@ package org.opensearch.repositories.blobstore; -import java.util.concurrent.CompletableFuture; +import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; @@ -44,11 +50,9 @@ import org.opensearch.common.CheckedFunction; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; -import org.opensearch.common.blobstore.stream.read.ReadContext; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; -import org.opensearch.common.io.InputStreamContainer; import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -63,22 +67,6 @@ import org.opensearch.gateway.CorruptStateException; import org.opensearch.index.store.exception.ChecksumCombinationException; import org.opensearch.snapshots.SnapshotInfo; -import org.opensearch.threadpool.ThreadPool; - -import java.io.IOException; -import java.io.InputStream; -import java.io.SequenceInputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.stream.Collectors; - -import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; /** * Snapshot metadata file format used in v2.0 and above @@ -129,16 +117,6 @@ public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunct return deserialize(blobName, namedXContentRegistry, Streams.readFully(blobContainer.readBlob(blobName))); } - public void readAsync(BlobContainer blobContainer, String name, NamedXContentRegistry namedXContentRegistry, ExecutorService executorService, ActionListener listener) throws IOException { - executorService.execute(() -> { - try { - listener.onResponse(read(blobContainer, name, namedXContentRegistry)); - } catch (Exception e) { - listener.onFailure(e); - } - }); - } - public String blobName(String name) { return String.format(Locale.ROOT, getBlobNameFormat(), name); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index a2fa691a16e4e..a79205f174c68 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -963,7 +963,7 @@ public void testReadLatestMetadataManifestSuccessButNoIndexMetadata() throws IOE remoteClusterStateService.start(); assertEquals( - remoteClusterStateService.getLatestClusterState(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()) + remoteClusterStateService.getLatestClusterState(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), false) .getMetadata() .getIndices() .size(), @@ -995,7 +995,8 @@ public void testReadLatestMetadataManifestSuccessButIndexMetadataFetchIOExceptio IllegalStateException.class, () -> remoteClusterStateService.getLatestClusterState( clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() + clusterState.metadata().clusterUUID(), + false ).getMetadata().getIndices() ); assertEquals(e.getMessage(), "Error while downloading IndexMetadata - " + uploadedIndexMetadata.getUploadedFilename()); @@ -1063,7 +1064,8 @@ public void testReadGlobalMetadata() throws IOException { ClusterState newClusterState = remoteClusterStateService.getLatestClusterState( clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() + clusterState.metadata().clusterUUID(), + false ); assertTrue(Metadata.isGlobalStateEquals(newClusterState.getMetadata(), expactedMetadata)); @@ -1108,7 +1110,8 @@ public void testReadGlobalMetadataIOException() throws IOException { IllegalStateException.class, () -> remoteClusterStateService.getLatestClusterState( clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() + clusterState.metadata().clusterUUID(), + false ) ); assertEquals(e.getMessage(), "Error while downloading Global Metadata - " + globalIndexMetadataName); @@ -1147,7 +1150,8 @@ public void testReadLatestIndexMetadataSuccess() throws IOException { Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( clusterState.getClusterName().value(), - clusterState.metadata().clusterUUID() + clusterState.metadata().clusterUUID(), + false ).getMetadata().getIndices(); assertEquals(indexMetadataMap.size(), 1); From 14a279df1a16f5d6076c1d1666ad7ee768a0278f Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 27 May 2024 15:43:43 +0530 Subject: [PATCH 098/133] Add transient settings to manifest Signed-off-by: Shivansh Arora --- .../opensearch/cluster/metadata/Metadata.java | 4 + .../remote/ClusterMetadataManifest.java | 41 ++++++- .../remote/ClusterStateDiffManifest.java | 20 ++++ .../remote/RemoteClusterStateService.java | 55 ++++++++- .../remote/RemoteClusterStateUtils.java | 4 + .../remote/RemoteGlobalMetadataManager.java | 7 ++ .../gateway/remote/RemoteManifestManager.java | 4 +- .../RemoteTransientSettingsMetadata.java | 108 ++++++++++++++++++ 8 files changed, 233 insertions(+), 10 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteTransientSettingsMetadata.java diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index d016501dd0910..14836371c7887 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -978,6 +978,10 @@ public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metad return metadata1.persistentSettings.equals(metadata2.persistentSettings); } + public static boolean isTransientSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.transientSettings.equals(metadata2.transientSettings); + } + public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) { return metadata1.templates.equals(metadata2.templates); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index aa179092949e3..863652c8492d6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -60,6 +60,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField METADATA_VERSION = new ParseField("metadata_version"); private static final ParseField UPLOADED_COORDINATOR_METADATA = new ParseField("uploaded_coordinator_metadata"); private static final ParseField UPLOADED_SETTINGS_METADATA = new ParseField("uploaded_settings_metadata"); + private static final ParseField UPLOADED_TRANSIENT_SETTINGS_METADATA = new ParseField("uploaded_transient_settings_metadata"); private static final ParseField UPLOADED_TEMPLATES_METADATA = new ParseField("uploaded_templates_metadata"); private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata"); private static final ParseField UPLOADED_DISCOVERY_NODES_METADATA = new ParseField("uploaded_discovery_nodes_metadata"); @@ -102,7 +103,8 @@ private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields .diffManifest(diffManifest(fields)) .routingTableVersion(routingTableVersion(fields)) .indicesRouting(indicesRouting(fields)) - .metadataVersion(metadataVersion(fields)); + .metadataVersion(metadataVersion(fields)) + .transientSettingsMetadata(transientSettingsMetadata(fields)); } private static long term(Object[] fields) { @@ -194,6 +196,10 @@ private static long metadataVersion(Object[] fields) { return (long) fields[20]; } + private static UploadedMetadataAttribute transientSettingsMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[21]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> manifestV0Builder(fields).build() @@ -288,6 +294,11 @@ private static void declareParser(ConstructingObjectParser indicesRouting, - long metadataVersion + long metadataVersion, + UploadedMetadataAttribute uploadedTransientSettingsMetadata ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -458,6 +475,7 @@ public ClusterMetadataManifest( this.routingTableVersion = routingTableVersion; this.indicesRouting = Collections.unmodifiableList(indicesRouting); this.metadataVersion = metadataVersion; + this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata; } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -488,6 +506,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.metadataVersion = in.readLong(); + this.uploadedTransientSettingsMetadata = new UploadedMetadataAttribute(in); } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); @@ -501,6 +520,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedClusterBlocksMetadata = null; this.diffManifest = null; this.metadataVersion = -1; + this.uploadedTransientSettingsMetadata = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; @@ -514,6 +534,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedClusterBlocksMetadata = null; this.diffManifest = null; this.metadataVersion = -1; + this.uploadedTransientSettingsMetadata = null; } } @@ -583,6 +604,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws getClusterBlocksMetadata().toXContent(builder, params); builder.endObject(); } + if (getTransientSettingsMetadata() != null) { + builder.startObject(UPLOADED_TRANSIENT_SETTINGS_METADATA.getPreferredName()); + getTransientSettingsMetadata().toXContent(builder, params); + builder.endObject(); + } if (getDiffManifest() != null) { builder.startObject(DIFF_MANIFEST.getPreferredName()); getDiffManifest().toXContent(builder, params); @@ -624,6 +650,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); out.writeLong(metadataVersion); + uploadedTransientSettingsMetadata.writeTo(out); } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); @@ -709,6 +736,7 @@ public static class Builder { private UploadedMetadataAttribute clusterBlocksMetadata; private UploadedMetadataAttribute coordinationMetadata; private UploadedMetadataAttribute settingsMetadata; + private UploadedMetadataAttribute transientSettingsMetadata; private UploadedMetadataAttribute templatesMetadata; private Map customMetadataMap; private int codecVersion; @@ -766,6 +794,11 @@ public Builder settingMetadata(UploadedMetadataAttribute settingsMetadata) { return this; } + public Builder transientSettingsMetadata(UploadedMetadataAttribute settingsMetadata) { + this.transientSettingsMetadata = settingsMetadata; + return this; + } + public Builder templatesMetadata(UploadedMetadataAttribute templatesMetadata) { this.templatesMetadata = templatesMetadata; return this; @@ -852,6 +885,7 @@ public Builder diffManifest(ClusterStateDiffManifest diffManifest) { public Builder() { indices = new ArrayList<>(); customMetadataMap = new HashMap<>(); + indicesRouting = new ArrayList<>(); } public Builder(ClusterMetadataManifest manifest) { @@ -899,7 +933,8 @@ public ClusterMetadataManifest build() { diffManifest, routingTableVersion, indicesRouting, - metadataVersion + metadataVersion, + transientSettingsMetadata ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 3e65c005a724c..a7dc3358766e5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -36,6 +36,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { private static final String METADATA_DIFF_FIELD = "metadata_diff"; private static final String COORDINATION_METADATA_UPDATED_FIELD = "coordination_metadata_diff"; private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff"; + private static final String TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD = "transient_settings_metadata_diff"; private static final String TEMPLATES_METADATA_UPDATED_FIELD = "templates_metadata_diff"; private static final String INDICES_DIFF_FIELD = "indices_diff"; private static final String METADATA_CUSTOM_DIFF_FIELD = "metadata_custom_diff"; @@ -48,6 +49,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { private final String toStateUUID; private final boolean coordinationMetadataUpdated; private final boolean settingsMetadataUpdated; + private final boolean transientSettingsMetadataUpdate; private final boolean templatesMetadataUpdated; private List customMetadataUpdated; private final List customMetadataDeleted; @@ -63,6 +65,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { toStateUUID = state.stateUUID(); coordinationMetadataUpdated = !Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); settingsMetadataUpdated = !Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); + transientSettingsMetadataUpdate = !Metadata.isTransientSettingsMetadataEqual(state.metadata(), previousState.metadata()); templatesMetadataUpdated = !Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); @@ -88,6 +91,7 @@ public ClusterStateDiffManifest(String fromStateUUID, String toStateUUID, boolean coordinationMetadataUpdated, boolean settingsMetadataUpdated, + boolean transientSettingsMetadataUpdate, boolean templatesMetadataUpdated, List customMetadataUpdated, List customMetadataDeleted, @@ -101,6 +105,7 @@ public ClusterStateDiffManifest(String fromStateUUID, this.toStateUUID = toStateUUID; this.coordinationMetadataUpdated = coordinationMetadataUpdated; this.settingsMetadataUpdated = settingsMetadataUpdated; + this.transientSettingsMetadataUpdate = transientSettingsMetadataUpdate; this.templatesMetadataUpdated = templatesMetadataUpdated; this.customMetadataUpdated = customMetadataUpdated; this.customMetadataDeleted = customMetadataDeleted; @@ -121,6 +126,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); + builder.field(TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD, transientSettingsMetadataUpdate); builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); builder.startObject(INDICES_DIFF_FIELD); builder.startArray(UPSERTS_FIELD); @@ -194,6 +200,9 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw case SETTINGS_METADATA_UPDATED_FIELD: builder.settingsMetadataUpdated(parser.booleanValue()); break; + case TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD: + builder.transientSettingsMetadataUpdate(parser.booleanValue()); + break; case TEMPLATES_METADATA_UPDATED_FIELD: builder.templatesMetadataUpdated(parser.booleanValue()); break; @@ -352,6 +361,10 @@ public boolean isSettingsMetadataUpdated() { return settingsMetadataUpdated; } + public boolean isTransientSettingsMetadataUpdated() { + return transientSettingsMetadataUpdate; + } + public boolean isTemplatesMetadataUpdated() { return templatesMetadataUpdated; } @@ -397,6 +410,7 @@ public static class Builder { private String toStateUUID; private boolean coordinationMetadataUpdated; private boolean settingsMetadataUpdated; + private boolean transientSettingsMetadataUpdated; private boolean templatesMetadataUpdated; private List customMetadataUpdated; private List customMetadataDeleted; @@ -428,6 +442,11 @@ public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { return this; } + public Builder transientSettingsMetadataUpdate(boolean settingsMetadataUpdated) { + this.transientSettingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { this.templatesMetadataUpdated = templatesMetadataUpdated; return this; @@ -479,6 +498,7 @@ public ClusterStateDiffManifest build() { toStateUUID, coordinationMetadataUpdated, settingsMetadataUpdated, + transientSettingsMetadataUpdated, templatesMetadataUpdated, customMetadataUpdated, customMetadataDeleted, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 09dc6da3287c7..3d95ad15023c0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -22,6 +22,7 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; @@ -213,7 +214,8 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true, true, true, - new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values())); + new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values()), + true); final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, @@ -221,6 +223,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri previousClusterUUID, uploadedMetadataResults.uploadedCoordinationMetadata, uploadedMetadataResults.uploadedSettingsMetadata, + uploadedMetadataResults.uploadedTransientSettingsMetadata, uploadedMetadataResults.uploadedTemplatesMetadata, uploadedMetadataResults.uploadedCustomMetadataMap, uploadedMetadataResults.uploadedDiscoveryNodes, @@ -324,6 +327,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( ; boolean updateSettingsMetadata = firstUploadForSplitGlobalMetadata || Metadata.isSettingsMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; + boolean updateTransientSettingsMetadata = firstUploadForSplitGlobalMetadata + || Metadata.isTransientSettingsMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; boolean updateTemplatesMetadata = firstUploadForSplitGlobalMetadata || Metadata.isTemplatesMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; // Write Global Metadata @@ -352,8 +357,9 @@ public ClusterMetadataManifest writeIncrementalMetadata( updateTemplatesMetadata, updateDiscoveryNodes, updateClusterBlocks, - indicesRoutingToUpload - ); + indicesRoutingToUpload, + updateTransientSettingsMetadata + ); // update the map if the metadata was uploaded @@ -376,6 +382,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousManifest.getPreviousClusterUUID(), updateCoordinationMetadata ? uploadedMetadataResults.uploadedCoordinationMetadata : previousManifest.getCoordinationMetadata(), updateSettingsMetadata ? uploadedMetadataResults.uploadedSettingsMetadata : previousManifest.getSettingsMetadata(), + updateTransientSettingsMetadata ? uploadedMetadataResults.uploadedTransientSettingsMetadata : previousManifest.getTransientSettingsMetadata(), updateTemplatesMetadata ? uploadedMetadataResults.uploadedTemplatesMetadata : previousManifest.getTemplatesMetadata(), firstUploadForSplitGlobalMetadata || !customsToUpload.isEmpty() ? allUploadedCustomMap @@ -453,9 +460,11 @@ private UploadedMetadataResults writeMetadataInParallel( boolean uploadTemplateMetadata, boolean uploadDiscoveryNodes, boolean uploadClusterBlock, - List indicesRoutingToUpload) throws IOException { + List indicesRoutingToUpload, + boolean uploadTransientSettingMetadata) throws IOException { int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata - ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size(); + ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size() + + (uploadTransientSettingMetadata ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new ConcurrentHashMap<>(totalUploadTasks); Map results = new ConcurrentHashMap<>(totalUploadTasks); @@ -487,6 +496,18 @@ private UploadedMetadataResults writeMetadataInParallel( ) ); } + if (uploadTransientSettingMetadata) { + uploadTasks.put( + TRANSIENT_SETTING_METADATA, + remoteGlobalMetadataManager.getAsyncMetadataWriteAction( + clusterState.metadata().transientSettings(), + clusterState.metadata().version(), + clusterState.metadata().clusterUUID(), + listener, + TRANSIENT_SETTING_METADATA + ) + ); + } if (uploadCoordinationMetadata) { uploadTasks.put( COORDINATION_METADATA, @@ -623,6 +644,8 @@ private UploadedMetadataResults writeMetadataInParallel( response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (SETTING_METADATA.equals(name)) { response.uploadedSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; + } else if (TRANSIENT_SETTING_METADATA.equals(name)) { + response.uploadedTransientSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (TEMPLATES_METADATA.equals(name)) { response.uploadedTemplatesMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (name.contains(UploadedIndexMetadata.COMPONENT_PREFIX)) { @@ -715,6 +738,7 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getPreviousClusterUUID(), previousManifest.getCoordinationMetadata(), previousManifest.getSettingsMetadata(), + previousManifest.getTransientSettingsMetadata(), previousManifest.getTemplatesMetadata(), previousManifest.getCustomMetadataMap(), previousManifest.getDiscoveryNodesMetadata(), @@ -828,12 +852,13 @@ private ClusterState readClusterStateInParallel( Map customToRead, boolean readCoordinationMetadata, boolean readSettingsMetadata, + boolean readTransientSettingsMetadata, boolean readTemplatesMetadata, boolean readDiscoveryNodes, boolean readClusterBlocks, List indicesRoutingToRead ) throws IOException { - int totalReadTasks = indicesToRead.size() + customToRead.size() + indicesRoutingToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + (readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0); + int totalReadTasks = indicesToRead.size() + customToRead.size() + indicesRoutingToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + (readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0) + (readTransientSettingsMetadata ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalReadTasks); List> asyncMetadataReadActions = new ArrayList<>(); List readResults = new ArrayList<>(); @@ -927,6 +952,19 @@ private ClusterState readClusterStateInParallel( ); } + if (readTransientSettingsMetadata) { + asyncMetadataReadActions.add( + remoteGlobalMetadataManager.getAsyncMetadataReadAction( + clusterName, + clusterUUID, + TRANSIENT_SETTING_METADATA, + TRANSIENT_SETTING_METADATA, + manifest.getTransientSettingsMetadata().getUploadedFilename(), + listener + ) + ); + } + if (readTemplatesMetadata) { asyncMetadataReadActions.add( remoteGlobalMetadataManager.getAsyncMetadataReadAction( @@ -1011,6 +1049,9 @@ private ClusterState readClusterStateInParallel( case SETTING_METADATA: metadataBuilder.persistentSettings((Settings) remoteReadResult.getObj()); break; + case TRANSIENT_SETTING_METADATA: + metadataBuilder.transientSettings((Settings) remoteReadResult.getObj()); + break; case TEMPLATES_METADATA: metadataBuilder.templates((TemplatesMetadata) remoteReadResult.getObj()); break; @@ -1052,6 +1093,7 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada manifest.getCustomMetadataMap(), manifest.getCoordinationMetadata() != null, manifest.getSettingsMetadata() != null, + manifest.getTransientSettingsMetadata() != null, manifest.getTemplatesMetadata() != null, includeEphemeral && manifest.getDiscoveryNodesMetadata() != null, includeEphemeral && manifest.getClusterBlocksMetadata() != null, @@ -1090,6 +1132,7 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata updatedCustomMetadata, diff.isCoordinationMetadataUpdated(), diff.isSettingsMetadataUpdated(), + diff.isTransientSettingsMetadataUpdated(), diff.isTemplatesMetadataUpdated(), diff.isDiscoveryNodesUpdated(), diff.isClusterBlocksUpdated(), diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index a383291637a9e..e34221c3aa53e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -77,6 +77,7 @@ public static class UploadedMetadataResults { Map uploadedCustomMetadataMap; ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes; ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks; @@ -87,6 +88,7 @@ public UploadedMetadataResults( Map uploadedCustomMetadataMap, ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes, ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks, @@ -96,6 +98,7 @@ public UploadedMetadataResults( this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; this.uploadedSettingsMetadata = uploadedSettingsMetadata; + this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata; this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; this.uploadedDiscoveryNodes = uploadedDiscoveryNodes; this.uploadedClusterBlocks = uploadedClusterBlocks; @@ -107,6 +110,7 @@ public UploadedMetadataResults() { this.uploadedCustomMetadataMap = new HashMap<>(); this.uploadedCoordinationMetadata = null; this.uploadedSettingsMetadata = null; + this.uploadedTransientSettingsMetadata = null; this.uploadedTemplatesMetadata = null; this.uploadedDiscoveryNodes = null; this.uploadedClusterBlocks = null; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index c74114509884f..4ea3a0cc97e50 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -12,6 +12,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; +import static org.opensearch.gateway.remote.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import java.io.IOException; import java.util.HashMap; @@ -134,6 +135,8 @@ CheckedRunnable getAsyncMetadataReadAction( remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else if (component.equals(SETTING_METADATA)) { remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } else if (component.equals(TRANSIENT_SETTING_METADATA)) { + remoteBlobStoreObject = new RemoteTransientSettingsMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else if (component.equals(CUSTOM_METADATA)) { remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else { @@ -291,6 +294,10 @@ private AbstractRemoteBlobStoreObject getRemoteObject(Object object, long metada return new RemoteCoordinationMetadata((CoordinationMetadata) object, metadataVersion, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else if (object instanceof Settings) { + if (customType != null && customType.equals(TRANSIENT_SETTING_METADATA)) { + return new RemoteTransientSettingsMetadata((Settings) object, metadataVersion, clusterUUID, blobStoreTransferService, + blobStoreRepository, clusterName, threadPool); + } return new RemotePersistentSettingsMetadata((Settings) object, metadataVersion, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } else if (object instanceof TemplatesMetadata) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index c6bc8a64293d1..60b847fe344fe 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -75,6 +75,7 @@ ClusterMetadataManifest uploadManifest( String previousClusterUUID, ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, Map uploadedCustomMetadataMap, ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodesMetadata, @@ -104,7 +105,8 @@ ClusterMetadataManifest uploadManifest( .diffManifest(clusterDiffManifest) .routingTableVersion(clusterState.getRoutingTable().version()) .indicesRouting(routingIndexMetadata) - .metadataVersion(clusterState.metadata().version()); + .metadataVersion(clusterState.metadata().version()) + .transientSettingsMetadata(uploadedTransientSettingsMetadata); final ClusterMetadataManifest manifest = manifestBuilder.build(); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest); return manifest; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteTransientSettingsMetadata.java new file mode 100644 index 0000000000000..6b3f072473ccd --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteTransientSettingsMetadata.java @@ -0,0 +1,108 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.common.io.Streams; +import org.opensearch.common.settings.Settings; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteTransientSettingsMetadata extends AbstractRemoteBlobStoreObject { + + public static final String TRANSIENT_SETTING_METADATA = "transient-settings"; + + public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "transient-settings", + METADATA_NAME_FORMAT, + Settings::fromXContent + ); + + private Settings transientSettings; + private long metadataVersion; + private String blobName; + private final String clusterUUID; + + public RemoteTransientSettingsMetadata(Settings transientSettings, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.transientSettings = transientSettings; + this.metadataVersion = metadataVersion; + this.clusterUUID = clusterUUID; + } + + public RemoteTransientSettingsMetadata(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.blobName = blobName; + this.clusterUUID = clusterUUID; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("global-metadata"), TRANSIENT_SETTING_METADATA); + } + + @Override + public String getFullBlobName() { + return blobName; + } + + @Override + public String generateBlobFileName() { + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + // setting the full blob path with name for future access + this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + return blobFileName; + } + + @Override + public Settings get() { + return transientSettings; + } + + @Override + public String clusterUUID() { + return clusterUUID; + } + + + @Override + public InputStream serialize() throws IOException { + return SETTINGS_METADATA_FORMAT.serialize(transientSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public Settings deserialize(InputStream inputStream) throws IOException { + return SETTINGS_METADATA_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, blobName); + } +} From 523889c9a3456f807f66482810861ac403117d40 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Tue, 28 May 2024 20:12:30 +0530 Subject: [PATCH 099/133] Create RemoteObjectStore class and corresponding implementations --- .../remote/RemoteRoutingTableService.java | 2 - .../RemoteClusterStateAttributesManager.java | 49 ++++---- .../RemoteClusterStateCleanupManager.java | 7 +- .../remote/RemoteClusterStateService.java | 12 +- .../remote/RemoteGlobalMetadataManager.java | 111 +++++++++++------- .../remote/RemoteIndexMetadataManager.java | 19 +-- .../gateway/remote/RemoteManifestManager.java | 16 +-- .../gateway/remote/RemoteObject.java | 27 ----- .../model/AbstractRemoteBlobObject.java | 70 +++++++++++ .../AbstractRemoteBlobStore.java} | 76 +++++------- .../{ => model}/BlobPathParameters.java | 10 +- .../{ => model}/RemoteClusterBlocks.java | 39 ++---- .../model/RemoteClusterBlocksBlobStore.java | 25 ++++ .../RemoteClusterMetadataManifest.java | 42 +++---- ...emoteClusterMetadataManifestBlobStore.java | 25 ++++ .../RemoteCoordinationMetadata.java | 37 ++---- .../RemoteCoordinationMetadataBlobStore.java | 25 ++++ .../{ => model}/RemoteCustomMetadata.java | 41 ++----- .../model/RemoteCustomMetadataBlobStore.java | 25 ++++ .../{ => model}/RemoteDiscoveryNodes.java | 37 ++---- .../model/RemoteDiscoveryNodesBlobStore.java | 25 ++++ .../{ => model}/RemoteIndexMetadata.java | 39 ++---- .../model/RemoteIndexMetadataBlobStore.java | 25 ++++ .../gateway/remote/model/RemoteObject.java | 42 +++++++ .../remote/model/RemoteObjectStore.java | 26 ++++ .../RemotePersistentSettingsBlobStore.java | 25 ++++ .../RemotePersistentSettingsMetadata.java | 41 ++----- .../remote/{ => model}/RemoteReadResult.java | 6 +- .../{ => model}/RemoteTemplatesMetadata.java | 44 +++---- .../RemoteTemplatesMetadataBlobStore.java | 25 ++++ .../RemoteTransientSettingsBlobStore.java | 25 ++++ .../RemoteTransientSettingsMetadata.java | 41 ++----- .../gateway/remote/model/package-info.java | 12 ++ ...RemoteClusterStateCleanupManagerTests.java | 9 +- .../RemoteClusterStateServiceTests.java | 8 +- .../remote/RemoteIndexMetadataTests.java | 36 +++--- .../remote/RemoteManifestManagerTests.java | 2 +- 37 files changed, 676 insertions(+), 450 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java rename server/src/main/java/org/opensearch/gateway/remote/{AbstractRemoteBlobStoreObject.java => model/AbstractRemoteBlobStore.java} (54%) rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/BlobPathParameters.java (74%) rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteClusterBlocks.java (71%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteClusterMetadataManifest.java (83%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteCoordinationMetadata.java (73%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteCustomMetadata.java (75%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteDiscoveryNodes.java (72%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteIndexMetadata.java (71%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataBlobStore.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteObject.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemotePersistentSettingsMetadata.java (71%) rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteReadResult.java (87%) rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteTemplatesMetadata.java (70%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataBlobStore.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/{ => model}/RemoteTransientSettingsMetadata.java (70%) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/package-info.java diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 266709894c1b7..5d105db4238d5 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -28,10 +28,8 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.index.Index; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateUtils; -import org.opensearch.gateway.remote.RemoteReadResult; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStream; import org.opensearch.gateway.remote.routingtable.IndexRoutingTableInputStreamReader; import org.opensearch.index.remote.RemoteStoreEnums; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 380fd5670558d..1130b92289cc2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -16,6 +16,12 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import org.opensearch.gateway.remote.model.AbstractRemoteBlobObject; +import org.opensearch.gateway.remote.model.RemoteClusterBlocks; +import org.opensearch.gateway.remote.model.RemoteClusterBlocksBlobStore; +import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes; +import org.opensearch.gateway.remote.model.RemoteDiscoveryNodesBlobStore; +import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; @@ -26,18 +32,21 @@ public class RemoteClusterStateAttributesManager { public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; public static final String DISCOVERY_NODES = "nodes"; public static final String CLUSTER_BLOCKS = "blocks"; - public static final String CUSTOM_PREFIX = "custom"; public static final int CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION = 1; private final BlobStoreTransferService blobStoreTransferService; private final BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; private final String clusterName; + private final RemoteClusterBlocksBlobStore clusterBlocksBlobStore; + private final RemoteDiscoveryNodesBlobStore discoveryNodesBlobStore; RemoteClusterStateAttributesManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository repository, ThreadPool threadPool, String clusterName) { this.blobStoreTransferService = blobStoreTransferService; this.blobStoreRepository = repository; this.threadPool = threadPool; this.clusterName = clusterName; + this.clusterBlocksBlobStore = new RemoteClusterBlocksBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.discoveryNodesBlobStore = new RemoteDiscoveryNodesBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } /** @@ -49,45 +58,41 @@ CheckedRunnable getAsyncMetadataWriteAction( ToXContent componentData, LatchedActionListener latchedActionListener ) { - AbstractRemoteBlobStoreObject remoteObject = getRemoteObject(componentData, clusterState.version(), clusterState.metadata().clusterUUID()); - ActionListener completionListener = ActionListener.wrap( - resp -> latchedActionListener.onResponse( - remoteObject.getUploadedMetadata() - ), - ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) - ); - return () -> remoteObject.writeAsync(completionListener); - } - - private AbstractRemoteBlobStoreObject getRemoteObject(ToXContent componentData, long stateVersion, String clusterUUID) { if (componentData instanceof DiscoveryNodes) { - return new RemoteDiscoveryNodes((DiscoveryNodes)componentData, stateVersion, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteDiscoveryNodes remoteObject = new RemoteDiscoveryNodes((DiscoveryNodes)componentData, clusterState.version(), clusterState.metadata().clusterUUID(), blobStoreRepository); + return () -> discoveryNodesBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else if (componentData instanceof ClusterBlocks) { - return new RemoteClusterBlocks((ClusterBlocks) componentData, stateVersion, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteClusterBlocks remoteObject = new RemoteClusterBlocks((ClusterBlocks) componentData, clusterState.version(), clusterState.metadata().clusterUUID(), blobStoreRepository); + return () -> clusterBlocksBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else { throw new RemoteStateTransferException("Remote object not found for "+ componentData.getClass()); } } + private ActionListener getActionListener(String component, AbstractRemoteBlobObject remoteObject, LatchedActionListener latchedActionListener) { + return ActionListener.wrap( + resp -> latchedActionListener.onResponse( + remoteObject.getUploadedMetadata() + ), + ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) + ); + } + public CheckedRunnable getAsyncMetadataReadAction( String clusterUUID, String component, String uploadedFilename, LatchedActionListener listener ) { - AbstractRemoteBlobStoreObject remoteObject = getRemoteObject(component, uploadedFilename, clusterUUID); ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); - return () -> remoteObject.readAsync(actionListener); - } - - private AbstractRemoteBlobStoreObject getRemoteObject(String component, String blobName, String clusterUUID) { if (component.equals(RemoteDiscoveryNodes.DISCOVERY_NODES)) { - return new RemoteDiscoveryNodes(blobName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(uploadedFilename, clusterUUID, blobStoreRepository); + return () -> discoveryNodesBlobStore.readAsync(remoteDiscoveryNodes, actionListener); } else if (component.equals(RemoteClusterBlocks.CLUSTER_BLOCKS)) { - return new RemoteClusterBlocks(blobName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(uploadedFilename, clusterUUID, blobStoreRepository); + return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); } else { throw new RemoteStateTransferException("Remote object not found for "+ component); } } - } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index d0ab53d337207..6e3e665d8bb38 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -24,11 +24,9 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import java.io.Closeable; import java.io.IOException; -import java.sql.Blob; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -36,10 +34,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_METADATA_FORMAT; -import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; /** * A Manager which provides APIs to clean up stale cluster state files and runs an async stale cleanup task diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 3d95ad15023c0..23e4b07b3ee28 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -21,8 +21,8 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; +import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; @@ -76,6 +76,7 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -795,7 +796,7 @@ public void start() { remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings,threadpool, getBlobStoreTransferService(), clusterName); remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings,threadpool, clusterName, getBlobStoreTransferService()); remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(getBlobStoreTransferService(), blobStoreRepository, threadpool, clusterName); - remoteManifestManager = new RemoteManifestManager(getBlobStoreTransferService(), blobStoreRepository, clusterSettings, nodeId, threadpool); + remoteManifestManager = new RemoteManifestManager(getBlobStoreTransferService(), blobStoreRepository, clusterSettings, nodeId, threadpool, clusterName); remoteClusterStateCleanupManager.start(); } @@ -916,7 +917,6 @@ private ClusterState readClusterStateInParallel( for (Map.Entry entry : customToRead.entrySet()) { asyncMetadataReadActions.add( remoteGlobalMetadataManager.getAsyncMetadataReadAction( - clusterName, clusterUUID, CUSTOM_METADATA, entry.getKey(), @@ -929,7 +929,6 @@ private ClusterState readClusterStateInParallel( if (readCoordinationMetadata) { asyncMetadataReadActions.add( remoteGlobalMetadataManager.getAsyncMetadataReadAction( - clusterName, clusterUUID, COORDINATION_METADATA, COORDINATION_METADATA, @@ -942,7 +941,6 @@ private ClusterState readClusterStateInParallel( if (readSettingsMetadata) { asyncMetadataReadActions.add( remoteGlobalMetadataManager.getAsyncMetadataReadAction( - clusterName, clusterUUID, SETTING_METADATA, SETTING_METADATA, @@ -955,7 +953,6 @@ private ClusterState readClusterStateInParallel( if (readTransientSettingsMetadata) { asyncMetadataReadActions.add( remoteGlobalMetadataManager.getAsyncMetadataReadAction( - clusterName, clusterUUID, TRANSIENT_SETTING_METADATA, TRANSIENT_SETTING_METADATA, @@ -968,7 +965,6 @@ private ClusterState readClusterStateInParallel( if (readTemplatesMetadata) { asyncMetadataReadActions.add( remoteGlobalMetadataManager.getAsyncMetadataReadAction( - clusterName, clusterUUID, TEMPLATES_METADATA, TEMPLATES_METADATA, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 4ea3a0cc97e50..ca186fa5711a6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -12,7 +12,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; -import static org.opensearch.gateway.remote.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; +import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import java.io.IOException; import java.util.HashMap; @@ -34,6 +34,18 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.gateway.remote.model.AbstractRemoteBlobObject; +import org.opensearch.gateway.remote.model.RemoteCoordinationMetadata; +import org.opensearch.gateway.remote.model.RemoteCoordinationMetadataBlobStore; +import org.opensearch.gateway.remote.model.RemoteCustomMetadata; +import org.opensearch.gateway.remote.model.RemoteCustomMetadataBlobStore; +import org.opensearch.gateway.remote.model.RemotePersistentSettingsBlobStore; +import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata; +import org.opensearch.gateway.remote.model.RemoteReadResult; +import org.opensearch.gateway.remote.model.RemoteTemplatesMetadata; +import org.opensearch.gateway.remote.model.RemoteTemplatesMetadataBlobStore; +import org.opensearch.gateway.remote.model.RemoteTransientSettingsBlobStore; +import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; @@ -89,6 +101,11 @@ public class RemoteGlobalMetadataManager { private volatile TimeValue globalMetadataUploadTimeout; private final BlobStoreTransferService blobStoreTransferService; private final String clusterName; + private final RemoteCoordinationMetadataBlobStore coordinationMetadataBlobStore; + private final RemoteTransientSettingsBlobStore transientSettingsBlobStore; + private final RemotePersistentSettingsBlobStore persistentSettingsBlobStore; + private final RemoteTemplatesMetadataBlobStore templatesMetadataBlobStore; + private final RemoteCustomMetadataBlobStore customMetadataBlobStore; RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, BlobStoreTransferService blobStoreTransferService, String clusterName) { @@ -97,6 +114,11 @@ public class RemoteGlobalMetadataManager { this.threadPool = threadPool; this.blobStoreTransferService = blobStoreTransferService; this.clusterName = clusterName; + this.coordinationMetadataBlobStore = new RemoteCoordinationMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.transientSettingsBlobStore = new RemoteTransientSettingsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.persistentSettingsBlobStore = new RemotePersistentSettingsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.templatesMetadataBlobStore = new RemoteTemplatesMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.customMetadataBlobStore = new RemoteCustomMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); } @@ -110,40 +132,66 @@ CheckedRunnable getAsyncMetadataWriteAction( LatchedActionListener latchedActionListener, String customType ) { - AbstractRemoteBlobStoreObject remoteBlobStoreObject = getRemoteObject(objectToUpload, metadataVersion, clusterUUID, customType); - ActionListener completionListener = ActionListener.wrap( + if (objectToUpload instanceof CoordinationMetadata) { + RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata((CoordinationMetadata) objectToUpload, metadataVersion, clusterUUID, + blobStoreRepository); + return () -> coordinationMetadataBlobStore.writeAsync(remoteCoordinationMetadata, getActionListener(remoteCoordinationMetadata, latchedActionListener)); + } else if (objectToUpload instanceof Settings) { + if (customType != null && customType.equals(TRANSIENT_SETTING_METADATA)) { + RemoteTransientSettingsMetadata remoteTransientSettingsMetadata = new RemoteTransientSettingsMetadata((Settings) objectToUpload, metadataVersion, clusterUUID, + blobStoreRepository); + return () -> transientSettingsBlobStore.writeAsync(remoteTransientSettingsMetadata, getActionListener(remoteTransientSettingsMetadata, latchedActionListener)); + } + RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata((Settings) objectToUpload, metadataVersion, clusterUUID, + blobStoreRepository); + return () -> persistentSettingsBlobStore.writeAsync(remotePersistentSettingsMetadata, getActionListener(remotePersistentSettingsMetadata, latchedActionListener)); + } else if (objectToUpload instanceof TemplatesMetadata) { + RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata((TemplatesMetadata) objectToUpload, metadataVersion, clusterUUID, + blobStoreRepository); + return () -> templatesMetadataBlobStore.writeAsync(remoteTemplatesMetadata, getActionListener(remoteTemplatesMetadata, latchedActionListener)); + } else if (objectToUpload instanceof Custom) { + RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata((Custom) objectToUpload, customType ,metadataVersion, clusterUUID, + blobStoreRepository); + return () -> customMetadataBlobStore.writeAsync(remoteCustomMetadata, getActionListener(remoteCustomMetadata, latchedActionListener)); + } + throw new RemoteStateTransferException("Remote object cannot be created for " + objectToUpload.getClass()); + } + + private ActionListener getActionListener(AbstractRemoteBlobObject remoteBlobStoreObject, LatchedActionListener latchedActionListener) { + return ActionListener.wrap( resp -> latchedActionListener.onResponse( remoteBlobStoreObject.getUploadedMetadata() ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException("Upload failed", ex)) ); - return () -> remoteBlobStoreObject.writeAsync(completionListener); } CheckedRunnable getAsyncMetadataReadAction( - String clusterName, String clusterUUID, String component, String componentName, String uploadFilename, LatchedActionListener listener ) { - AbstractRemoteBlobStoreObject remoteBlobStoreObject; + ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); if (component.equals(COORDINATION_METADATA)) { - remoteBlobStoreObject = new RemoteCoordinationMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteCoordinationMetadata remoteBlobStoreObject = new RemoteCoordinationMetadata(uploadFilename, clusterUUID, blobStoreRepository); + return () -> coordinationMetadataBlobStore.readAsync(remoteBlobStoreObject, actionListener); } else if (component.equals(TEMPLATES_METADATA)) { - remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteTemplatesMetadata remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, blobStoreRepository); + return () -> templatesMetadataBlobStore.readAsync(remoteBlobStoreObject, actionListener); } else if (component.equals(SETTING_METADATA)) { - remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemotePersistentSettingsMetadata remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, blobStoreRepository); + return () -> persistentSettingsBlobStore.readAsync(remoteBlobStoreObject, actionListener); } else if (component.equals(TRANSIENT_SETTING_METADATA)) { - remoteBlobStoreObject = new RemoteTransientSettingsMetadata(uploadFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteTransientSettingsMetadata remoteBlobStoreObject = new RemoteTransientSettingsMetadata(uploadFilename, clusterUUID, blobStoreRepository); + return () -> transientSettingsBlobStore.readAsync(remoteBlobStoreObject, actionListener); } else if (component.equals(CUSTOM_METADATA)) { - remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteCustomMetadata remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, blobStoreRepository); + return () -> customMetadataBlobStore.readAsync(remoteBlobStoreObject, actionListener); } else { throw new RemoteStateTransferException("Unknown component " + componentName); } - ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); - return () -> remoteBlobStoreObject.readAsync(actionListener); } Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { @@ -201,8 +249,8 @@ public CoordinationMetadata getCoordinationMetadata(String clusterUUID, String c // Fetch Coordination metadata if (coordinationMetadataFileName != null) { RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(coordinationMetadataFileName, clusterUUID, - blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - return remoteCoordinationMetadata.read(); + blobStoreRepository); + return coordinationMetadataBlobStore.read(remoteCoordinationMetadata); } else { return CoordinationMetadata.EMPTY_METADATA; } @@ -219,8 +267,8 @@ public Settings getSettingsMetadata(String clusterUUID, String settingsMetadataF // Fetch Settings metadata if (settingsMetadataFileName != null) { RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(settingsMetadataFileName, clusterUUID, - blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - return remotePersistentSettingsMetadata.read(); + blobStoreRepository); + return persistentSettingsBlobStore.read(remotePersistentSettingsMetadata); } else { return Settings.EMPTY; } @@ -237,8 +285,8 @@ public TemplatesMetadata getTemplatesMetadata(String clusterUUID, String templat // Fetch Templates metadata if (templatesMetadataFileName != null) { RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(templatesMetadataFileName, clusterUUID, - blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - return remoteTemplatesMetadata.read(); + blobStoreRepository); + return templatesMetadataBlobStore.read(remoteTemplatesMetadata); } else { return TemplatesMetadata.EMPTY_METADATA; } @@ -254,8 +302,8 @@ public Metadata.Custom getCustomsMetadata(String clusterUUID, String customMetad requireNonNull(customMetadataFileName); try { // Fetch Custom metadata - RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - return remoteCustomMetadata.read(); + RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, blobStoreRepository); + return customMetadataBlobStore.read(remoteCustomMetadata); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), @@ -289,27 +337,6 @@ Map getUpdatedCustoms(ClusterState currentState, Cluste return updatedCustom; } - private AbstractRemoteBlobStoreObject getRemoteObject(Object object, long metadataVersion, String clusterUUID, String customType) { - if (object instanceof CoordinationMetadata) { - return new RemoteCoordinationMetadata((CoordinationMetadata) object, metadataVersion, clusterUUID, blobStoreTransferService, - blobStoreRepository, clusterName, threadPool); - } else if (object instanceof Settings) { - if (customType != null && customType.equals(TRANSIENT_SETTING_METADATA)) { - return new RemoteTransientSettingsMetadata((Settings) object, metadataVersion, clusterUUID, blobStoreTransferService, - blobStoreRepository, clusterName, threadPool); - } - return new RemotePersistentSettingsMetadata((Settings) object, metadataVersion, clusterUUID, blobStoreTransferService, - blobStoreRepository, clusterName, threadPool); - } else if (object instanceof TemplatesMetadata) { - return new RemoteTemplatesMetadata((TemplatesMetadata) object, metadataVersion, clusterUUID, blobStoreTransferService, - blobStoreRepository, clusterName, threadPool); - } else if (object instanceof Custom) { - return new RemoteCustomMetadata((Custom) object, customType ,metadataVersion, clusterUUID, blobStoreTransferService, - blobStoreRepository, clusterName, threadPool); - } - throw new RemoteStateTransferException("Remote object cannot be created for " + object.getClass()); - } - private BlobContainer globalMetadataContainer(String clusterName, String clusterUUID) { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/ return blobStoreRepository.blobStore() diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index 5f5459994e51e..67825546d54b2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -9,7 +9,7 @@ package org.opensearch.gateway.remote; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; import java.io.IOException; import java.util.Locale; @@ -20,6 +20,9 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.gateway.remote.model.RemoteIndexMetadata; +import org.opensearch.gateway.remote.model.RemoteIndexMetadataBlobStore; +import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; @@ -39,12 +42,14 @@ public class RemoteIndexMetadataManager { private final ThreadPool threadPool; private final BlobStoreTransferService blobStoreTransferService; + private final RemoteIndexMetadataBlobStore indexMetadataBlobStore; private volatile TimeValue indexMetadataUploadTimeout; private final String clusterName; public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, String clusterName, BlobStoreTransferService blobStoreTransferService) { + indexMetadataBlobStore = new RemoteIndexMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.blobStoreRepository = blobStoreRepository; this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); this.threadPool = threadPool; @@ -61,14 +66,14 @@ public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, Clust */ CheckedRunnable getIndexMetadataAsyncAction(IndexMetadata indexMetadata, String clusterUUID, LatchedActionListener latchedActionListener) { - RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( remoteIndexMetadata.getUploadedMetadata() ), ex -> latchedActionListener.onFailure(new RemoteStateTransferException(indexMetadata.getIndex().getName(), ex)) ); - return () -> remoteIndexMetadata.writeAsync(completionListener); + return () -> indexMetadataBlobStore.writeAsync(remoteIndexMetadata, completionListener); } CheckedRunnable getAsyncIndexMetadataReadAction( @@ -76,11 +81,11 @@ CheckedRunnable getAsyncIndexMetadataReadAction( String uploadedFilename, LatchedActionListener latchedActionListener ) { - RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, blobStoreRepository); ActionListener actionListener = ActionListener.wrap( response -> latchedActionListener.onResponse(new RemoteReadResult(response, INDEX_PATH_TOKEN, response.getIndexName())), latchedActionListener::onFailure); - return () -> remoteIndexMetadata.readAsync(actionListener); + return () -> indexMetadataBlobStore.readAsync(remoteIndexMetadata, actionListener); } /** @@ -93,9 +98,9 @@ IndexMetadata getIndexMetadata( ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata, String clusterUUID, int manifestCodecVersion ) { RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(RemoteClusterStateUtils.getFormattedFileName( - uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, blobStoreRepository); try { - return remoteIndexMetadata.read(); + return indexMetadataBlobStore.read(remoteIndexMetadata); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading IndexMetadata - %s", uploadedIndexMetadata.getUploadedFilename()), diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 60b847fe344fe..0bd89c73003ed 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -20,10 +20,11 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; +import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifestBlobStore; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import java.io.IOException; import java.util.HashMap; @@ -38,7 +39,6 @@ import org.opensearch.threadpool.ThreadPool; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; @@ -58,14 +58,16 @@ public class RemoteManifestManager { private volatile TimeValue metadataManifestUploadTimeout; private final String nodeId; private final ThreadPool threadPool; + private final RemoteClusterMetadataManifestBlobStore manifestBlobStore; private static final Logger logger = LogManager.getLogger(RemoteManifestManager.class); - RemoteManifestManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, String nodeId, ThreadPool threadPool) { + RemoteManifestManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, String nodeId, ThreadPool threadPool, String clusterName) { this.blobStoreTransferService = blobStoreTransferService; this.blobStoreRepository = blobStoreRepository; this.metadataManifestUploadTimeout = clusterSettings.get(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING); this.nodeId = nodeId; this.threadPool = threadPool; + manifestBlobStore = new RemoteClusterMetadataManifestBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); } @@ -124,8 +126,8 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); }, ex -> { exceptionReference.set(ex); }), latch); - RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(uploadManifest, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - remoteClusterMetadataManifest.writeAsync(completionListener); + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(uploadManifest, clusterUUID, blobStoreRepository); + manifestBlobStore.writeAsync(remoteClusterMetadataManifest, completionListener); try { if (latch.await(getMetadataManifestUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) { @@ -188,8 +190,8 @@ ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, S throws IllegalStateException { try { String fullBlobName = getManifestFolderPath(clusterName, clusterUUID).buildAsString() + filename; - RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(fullBlobName, clusterUUID, blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - return remoteClusterMetadataManifest.read(); + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(fullBlobName, clusterUUID, blobStoreRepository); + return manifestBlobStore.read(remoteClusterMetadataManifest); } catch (IOException e) { throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java deleted file mode 100644 index 34af997206e84..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteObject.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote; - -import java.io.IOException; -import java.io.InputStream; -import org.opensearch.common.CheckedRunnable; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; - -public interface RemoteObject { - public T get(); - public String clusterUUID(); - public InputStream serialize() throws IOException; - public T deserialize(InputStream inputStream) throws IOException; - - public void writeAsync(ActionListener listener); - public T read() throws IOException; - public void readAsync(ActionListener listener); - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java new file mode 100644 index 0000000000000..5ab621406bc85 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; + +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.core.compress.Compressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.repositories.blobstore.BlobStoreRepository; + +/** + * An extension of {@link RemoteObject} class which caters to the use case of writing to and reading from a blob storage + * + * @param The class type which can be uploaded to or downloaded from a blob storage. + */ +public abstract class AbstractRemoteBlobObject implements RemoteObject { + + protected String blobFileName; + + protected String blobName; + private final BlobStoreRepository blobStoreRepository; + private final String clusterUUID; + + public AbstractRemoteBlobObject(BlobStoreRepository blobStoreRepository, String clusterUUID) { + this.blobStoreRepository = blobStoreRepository; + this.clusterUUID = clusterUUID; + } + + public abstract BlobPathParameters getBlobPathParameters(); + + public String getFullBlobName() { + return blobName; + } + + public String getBlobFileName() { + if (blobFileName == null) { + String[] pathTokens = blobName.split(PATH_DELIMITER); + blobFileName = pathTokens[pathTokens.length - 1]; + } + return blobFileName; + } + + public abstract String generateBlobFileName(); + + public String clusterUUID() { + return clusterUUID; + } + + public abstract UploadedMetadata getUploadedMetadata(); + + protected void setFullBlobName(BlobPath blobPath) { + this.blobName = blobPath.buildAsString() + blobFileName; + } + + protected Compressor getCompressor() { + return blobStoreRepository.getCompressor(); + } + + protected BlobStoreRepository getBlobStoreRepository() { + return this.blobStoreRepository; + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java similarity index 54% rename from server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java rename to server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java index b826e8999f877..d873fc4eed05a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/AbstractRemoteBlobStoreObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; @@ -14,24 +14,28 @@ import java.io.InputStream; import java.util.Arrays; import java.util.concurrent.ExecutorService; -import org.opensearch.common.CheckedRunnable; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.compress.Compressor; -import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; -public abstract class AbstractRemoteBlobStoreObject implements RemoteObject { +/** + * Abstract class for a blob type storage + * + * @param The entity which can be uploaded to / downloaded from blob store + * @param The concrete class implementing {@link RemoteObject} which is used as a wrapper for T entity. + */ +public class AbstractRemoteBlobStore> implements RemoteObjectStore { private final BlobStoreTransferService transferService; private final BlobStoreRepository blobStoreRepository; private final String clusterName; private final ExecutorService executorService; - public AbstractRemoteBlobStoreObject(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + public AbstractRemoteBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { this.transferService = blobStoreTransferService; this.blobStoreRepository = blobStoreRepository; @@ -39,61 +43,51 @@ public AbstractRemoteBlobStoreObject(BlobStoreTransferService blobStoreTransferS this.executorService = threadPool.executor(ThreadPool.Names.GENERIC); } - public abstract BlobPathParameters getBlobPathParameters(); - - public abstract String getFullBlobName(); - - public String getBlobFileName() { - if (getFullBlobName() == null) { - generateBlobFileName(); - } - String[] pathTokens = getFullBlobName().split(PATH_DELIMITER); - return pathTokens[pathTokens.length - 1]; - } - - public abstract String generateBlobFileName(); - - public abstract UploadedMetadata getUploadedMetadata(); - @Override - public void writeAsync(ActionListener listener) { - assert get() != null; + public void writeAsync(RemoteObject obj, ActionListener listener) { + assert obj instanceof AbstractRemoteBlobObject; + AbstractRemoteBlobObject blobStoreObj = (AbstractRemoteBlobObject) obj; + assert blobStoreObj.get() != null; try { - InputStream inputStream = serialize(); - transferService.uploadBlob(inputStream, getBlobPathForUpload(), getBlobFileName(), WritePriority.URGENT, listener); + InputStream inputStream = obj.serialize(); + BlobPath blobPath = getBlobPathForUpload(blobStoreObj); + blobStoreObj.setFullBlobName(blobPath); + transferService.uploadBlob(inputStream, getBlobPathForUpload(blobStoreObj), blobStoreObj.getBlobFileName(), WritePriority.URGENT, listener); } catch (Exception e) { listener.onFailure(e); } } @Override - public T read() throws IOException { - assert getFullBlobName() != null; - return deserialize( - transferService.downloadBlob(getBlobPathForDownload(), getBlobFileName())); + public T read(RemoteObject obj) throws IOException { + assert obj instanceof AbstractRemoteBlobObject; + AbstractRemoteBlobObject blobStoreObj = (AbstractRemoteBlobObject) obj; + assert blobStoreObj.getFullBlobName() != null; + return blobStoreObj.deserialize( + transferService.downloadBlob(getBlobPathForDownload(blobStoreObj), blobStoreObj.getBlobFileName())); } @Override - public void readAsync(ActionListener listener) { + public void readAsync(RemoteObject obj, ActionListener listener) { executorService.execute(() -> { try { - listener.onResponse(read()); + listener.onResponse(read(obj)); } catch (Exception e) { listener.onFailure(e); } }); } - public BlobPath getBlobPathForUpload() { - BlobPath blobPath = blobStoreRepository.basePath().add(RemoteClusterStateUtils.encodeString(clusterName)).add("cluster-state").add(clusterUUID()); - for (String token : getBlobPathParameters().getPathTokens()) { + private BlobPath getBlobPathForUpload(AbstractRemoteBlobObject obj) { + BlobPath blobPath = blobStoreRepository.basePath().add(RemoteClusterStateUtils.encodeString(clusterName)).add("cluster-state").add(obj.clusterUUID()); + for (String token : obj.getBlobPathParameters().getPathTokens()) { blobPath = blobPath.add(token); } return blobPath; } - public BlobPath getBlobPathForDownload() { - String[] pathTokens = extractBlobPathTokens(getFullBlobName()); + private BlobPath getBlobPathForDownload(AbstractRemoteBlobObject obj) { + String[] pathTokens = extractBlobPathTokens(obj.getFullBlobName()); BlobPath blobPath = new BlobPath(); for (String token : pathTokens) { blobPath = blobPath.add(token); @@ -101,14 +95,6 @@ public BlobPath getBlobPathForDownload() { return blobPath; } - protected Compressor getCompressor() { - return blobStoreRepository.getCompressor(); - } - - protected BlobStoreRepository getBlobStoreRepository() { - return this.blobStoreRepository; - } - private static String[] extractBlobPathTokens(String blobName) { String[] blobNameTokens = blobName.split(PATH_DELIMITER); return Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1); diff --git a/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java b/server/src/main/java/org/opensearch/gateway/remote/model/BlobPathParameters.java similarity index 74% rename from server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java rename to server/src/main/java/org/opensearch/gateway/remote/model/BlobPathParameters.java index 4b47bd744ef1a..cc7a2c18b8792 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/BlobPathParameters.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/BlobPathParameters.java @@ -6,14 +6,18 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import java.util.List; +/** + * Parameters which can be used to construct a blob path + * + */ public class BlobPathParameters { - private List pathTokens; - private String filePrefix; + private final List pathTokens; + private final String filePrefix; public BlobPathParameters(List pathTokens, String filePrefix) { this.pathTokens = pathTokens; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java similarity index 71% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteClusterBlocks.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index 8239ca76d3b6b..f3906321301c9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -19,13 +19,15 @@ import org.opensearch.common.io.Streams; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteClusterBlocks extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading {@link ClusterBlocks} to/from remote blob store + */ +public class RemoteClusterBlocks extends AbstractRemoteBlobObject { public static final String CLUSTER_BLOCKS = "blocks"; public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( @@ -36,24 +38,17 @@ public class RemoteClusterBlocks extends AbstractRemoteBlobStoreObject______ @@ -76,8 +66,7 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -92,10 +81,6 @@ public ClusterBlocks get() { return clusterBlocks; } - @Override - public String clusterUUID() { - return clusterUUID; - } @Override public InputStream serialize() throws IOException { diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java new file mode 100644 index 0000000000000..31aede0b78dda --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link ClusterBlocks} to/from blob store + */ +public class RemoteClusterBlocksBlobStore extends AbstractRemoteBlobStore { + + public RemoteClusterBlocksBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java similarity index 83% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteClusterMetadataManifest.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index 54e160e961671..5b4e10e005be2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -14,15 +14,18 @@ import java.io.InputStream; import java.util.List; import org.opensearch.common.io.Streams; +import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteClusterMetadataManifest extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading {@link ClusterMetadataManifest} to/from remote blob store + */ +public class RemoteClusterMetadataManifest extends AbstractRemoteBlobObject { public static final String MANIFEST_PATH_TOKEN = "manifest"; public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; @@ -52,23 +55,17 @@ public class RemoteClusterMetadataManifest extends AbstractRemoteBlobStoreObject ); private ClusterMetadataManifest clusterMetadataManifest; - private String blobName; - private final String clusterUUID; - public RemoteClusterMetadataManifest(ClusterMetadataManifest clusterMetadataManifest, String clusterUUID, BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemoteClusterMetadataManifest(ClusterMetadataManifest clusterMetadataManifest, String clusterUUID, + BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.clusterMetadataManifest = clusterMetadataManifest; - this.clusterUUID = clusterUUID; } - public RemoteClusterMetadataManifest(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemoteClusterMetadataManifest(String blobName, String clusterUUID, + BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.blobName = blobName; - this.clusterUUID = clusterUUID; } @Override @@ -76,11 +73,6 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(MANIFEST_PATH_TOKEN), MANIFEST_FILE_PREFIX); } - @Override - public String getFullBlobName() { - return blobName; - } - @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/manifest/manifest______C/P____ @@ -95,8 +87,7 @@ public String generateBlobFileName() { String.valueOf(clusterMetadataManifest.getCodecVersion()) // Keep the codec version at last place only, during read we reads last place to // determine codec version. ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -110,11 +101,6 @@ public ClusterMetadataManifest get() { return clusterMetadataManifest; } - @Override - public String clusterUUID() { - return clusterUUID; - } - @Override public InputStream serialize() throws IOException { return CLUSTER_METADATA_MANIFEST_FORMAT.serialize(clusterMetadataManifest, generateBlobFileName(), getCompressor(), diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java new file mode 100644 index 0000000000000..09ed9046e9087 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link ClusterMetadataManifest} to/from blob store + */ +public class RemoteClusterMetadataManifestBlobStore extends AbstractRemoteBlobStore{ + + public RemoteClusterMetadataManifestBlobStore(BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java similarity index 73% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java index d3d53aab9a4b8..76f97cd98c796 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteCoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; @@ -19,13 +19,15 @@ import org.opensearch.common.io.Streams; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteCoordinationMetadata extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading {@link CoordinationMetadata} to/from remote blob store + */ +public class RemoteCoordinationMetadata extends AbstractRemoteBlobObject { public static final String COORDINATION_METADATA = "coordination"; public static final ChecksumBlobStoreFormat COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( @@ -36,22 +38,16 @@ public class RemoteCoordinationMetadata extends AbstractRemoteBlobStoreObject______ @@ -74,8 +65,7 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -84,11 +74,6 @@ public CoordinationMetadata get() { return coordinationMetadata; } - @Override - public String clusterUUID() { - return clusterUUID; - } - @Override public InputStream serialize() throws IOException { return COORDINATION_METADATA_FORMAT.serialize(coordinationMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataBlobStore.java new file mode 100644 index 0000000000000..1790d97ccca2f --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link CoordinationMetadata} to/from blob store + */ +public class RemoteCoordinationMetadataBlobStore extends AbstractRemoteBlobStore { + + public RemoteCoordinationMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java similarity index 75% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java index f70a99f2fa21c..81e7794fc6949 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteCustomMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; @@ -20,13 +20,15 @@ import org.opensearch.common.io.Streams; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteCustomMetadata extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading {@link Custom} to/from remote blob store + */ +public class RemoteCustomMetadata extends AbstractRemoteBlobObject { public static final String CUSTOM_METADATA = "custom"; public static final String CUSTOM_DELIMITER = "--"; @@ -41,17 +43,13 @@ public class RemoteCustomMetadata extends AbstractRemoteBlobStoreObject private Custom custom; private final String customType; private long metadataVersion; - private String blobName; - private final String clusterUUID; - public RemoteCustomMetadata(Custom custom, String customType, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemoteCustomMetadata(Custom custom, String customType, long metadataVersion, String clusterUUID, + BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.custom = custom; this.customType = customType; this.metadataVersion = metadataVersion; - this.clusterUUID = clusterUUID; this.customBlobStoreFormat = new ChecksumBlobStoreFormat<>( "custom", METADATA_NAME_FORMAT, @@ -59,13 +57,11 @@ public RemoteCustomMetadata(Custom custom, String customType, long metadataVersi ); } - public RemoteCustomMetadata(String blobName, String customType, String clusterUUID, BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemoteCustomMetadata(String blobName, String customType, String clusterUUID, + BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.blobName = blobName; this.customType = customType; - this.clusterUUID = clusterUUID; this.customBlobStoreFormat = new ChecksumBlobStoreFormat<>( "custom", METADATA_NAME_FORMAT, @@ -79,11 +75,6 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of("global-metadata"), prefix); } - @Override - public String getFullBlobName() { - return blobName; - } - @Override public String generateBlobFileName() { // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ @@ -95,8 +86,7 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -105,11 +95,6 @@ public Custom get() { return custom; } - @Override - public String clusterUUID() { - return clusterUUID; - } - @Override public InputStream serialize() throws IOException { return customBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java new file mode 100644 index 0000000000000..1d24e014433d2 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.metadata.Metadata.Custom; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link Custom} to/from blob store + */ +public class RemoteCustomMetadataBlobStore extends AbstractRemoteBlobStore { + + public RemoteCustomMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java similarity index 72% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteDiscoveryNodes.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index cfa74f683ac4f..568fce6ac0007 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -19,13 +19,15 @@ import org.opensearch.common.io.Streams; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteDiscoveryNodes extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading {@link DiscoveryNodes} to/from remote blob store + */ +public class RemoteDiscoveryNodes extends AbstractRemoteBlobObject { public static final String DISCOVERY_NODES = "nodes"; public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( @@ -36,22 +38,16 @@ public class RemoteDiscoveryNodes extends AbstractRemoteBlobStoreObject______ @@ -74,8 +65,7 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -90,11 +80,6 @@ public DiscoveryNodes get() { return discoveryNodes; } - @Override - public String clusterUUID() { - return clusterUUID; - } - @Override public InputStream serialize() throws IOException { return DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesBlobStore.java new file mode 100644 index 0000000000000..68da66665a73f --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link DiscoveryNodes} to/from blob store + */ +public class RemoteDiscoveryNodesBlobStore extends AbstractRemoteBlobStore { + + public RemoteDiscoveryNodesBlobStore(BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java similarity index 71% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java index b177747c8a249..983bc81f6c455 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_PLAIN_FORMAT; @@ -18,13 +18,15 @@ import org.opensearch.common.io.Streams; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteIndexMetadata extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading {@link IndexMetadata} to/from remote blob store + */ +public class RemoteIndexMetadata extends AbstractRemoteBlobObject { public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; @@ -36,23 +38,15 @@ public class RemoteIndexMetadata extends AbstractRemoteBlobStoreObject { + public RemoteIndexMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, + ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObject.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObject.java new file mode 100644 index 0000000000000..9fb4738ff0efa --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObject.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import java.io.IOException; +import java.io.InputStream; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; + +/** + * An interface to read/write and object from/to a remote storage. This interface is agnostic of the remote storage type. + * + * @param The object type which can be upload to or download from remote storage. + */ +public interface RemoteObject { + + /** + * @return The entity T contained within this class + */ + public T get(); + + /** + * @return An InputStream created by serializing the entity T + * @throws IOException Exception encountered while serialization + */ + public InputStream serialize() throws IOException; + + /** + * @param inputStream The InputStream which is used to read the serialized entity + * @return The entity T after deserialization + * @throws IOException Exception encountered while deserialization + */ + public T deserialize(InputStream inputStream) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java new file mode 100644 index 0000000000000..b8ed8ebb7fa0c --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import java.io.IOException; +import org.opensearch.core.action.ActionListener; + +/** + * An interface to read/write an object from/to a remote storage. This interface is agnostic of the remote storage type. + * + * @param The object type which can be uploaded to or downloaded from remote storage. + */ +public interface RemoteObjectStore> { + + public void writeAsync(RemoteObject obj, ActionListener listener); + + public T read(RemoteObject obj) throws IOException; + + public void readAsync(RemoteObject obj, ActionListener listener); +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java new file mode 100644 index 0000000000000..ff63cd528fc2c --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.common.settings.Settings; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download persistent {@link Settings} to/from blob store + */ +public class RemotePersistentSettingsBlobStore extends AbstractRemoteBlobStore { + + public RemotePersistentSettingsBlobStore(BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java similarity index 71% rename from server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java index f23cd6dcc1eb8..7dcbac745b49a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemotePersistentSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; @@ -19,13 +19,15 @@ import org.opensearch.common.settings.Settings; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading persistent {@link Settings} to/from remote blob store + */ +public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobObject { public static final String SETTING_METADATA = "settings"; @@ -37,22 +39,16 @@ public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobStoreObj private Settings persistentSettings; private long metadataVersion; - private String blobName; - private final String clusterUUID; - public RemotePersistentSettingsMetadata(Settings settings, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemotePersistentSettingsMetadata(Settings settings, long metadataVersion, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.persistentSettings = settings; this.metadataVersion = metadataVersion; - this.clusterUUID = clusterUUID; } - public RemotePersistentSettingsMetadata(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemotePersistentSettingsMetadata(String blobName, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.blobName = blobName; - this.clusterUUID = clusterUUID; } @Override @@ -60,11 +56,6 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of("global-metadata"), SETTING_METADATA); } - @Override - public String getFullBlobName() { - return blobName; - } - @Override public String generateBlobFileName() { String blobFileName = String.join( @@ -74,8 +65,7 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -84,15 +74,10 @@ public Settings get() { return persistentSettings; } - @Override - public String clusterUUID() { - return clusterUUID; - } - - @Override public InputStream serialize() throws IOException { - return SETTINGS_METADATA_FORMAT.serialize(persistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return SETTINGS_METADATA_FORMAT.serialize(persistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) + .streamInput(); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteReadResult.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java similarity index 87% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteReadResult.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java index dea8a6221d05a..adee09eaeffef 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteReadResult.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java @@ -6,11 +6,15 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import org.opensearch.core.xcontent.ToXContent; +/** + * Container class for entity read from remote store + */ public class RemoteReadResult { + ToXContent obj; String component; String componentName; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadata.java similarity index 70% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadata.java index 9c302993536a9..363db7fcabf2a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteTemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadata.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; @@ -19,13 +19,15 @@ import org.opensearch.common.io.Streams; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteTemplatesMetadata extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading {@link TemplatesMetadata} to/from remote blob store + */ +public class RemoteTemplatesMetadata extends AbstractRemoteBlobObject { public static final String TEMPLATES_METADATA = "templates"; @@ -36,21 +38,16 @@ public class RemoteTemplatesMetadata extends AbstractRemoteBlobStoreObject______ + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/______ + // String blobFileName = String.join( DELIMITER, getBlobPathParameters().getFilePrefix(), @@ -73,8 +66,7 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -83,14 +75,10 @@ public TemplatesMetadata get() { return templatesMetadata; } - @Override - public String clusterUUID() { - return clusterUUID; - } - @Override public InputStream serialize() throws IOException { - return TEMPLATES_METADATA_FORMAT.serialize(templatesMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return TEMPLATES_METADATA_FORMAT.serialize(templatesMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) + .streamInput(); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataBlobStore.java new file mode 100644 index 0000000000000..94a0b4f1bf365 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link TemplatesMetadata} to/from blob store + */ +public class RemoteTemplatesMetadataBlobStore extends AbstractRemoteBlobStore { + + public RemoteTemplatesMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java new file mode 100644 index 0000000000000..5004cef8a9c66 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.common.settings.Settings; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +/** + * An implementation of {@link RemoteObjectStore} which is used to upload/download transient {@link Settings} to/from blob store + */ +public class RemoteTransientSettingsBlobStore extends AbstractRemoteBlobStore { + + public RemoteTransientSettingsBlobStore(BlobStoreTransferService blobStoreTransferService, + BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java similarity index 70% rename from server/src/main/java/org/opensearch/gateway/remote/RemoteTransientSettingsMetadata.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java index 6b3f072473ccd..ee0116a54bbae 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteTransientSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; @@ -19,13 +19,15 @@ import org.opensearch.common.settings.Settings; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; -public class RemoteTransientSettingsMetadata extends AbstractRemoteBlobStoreObject { +/** + * Wrapper class for uploading/downloading transient {@link Settings} to/from remote blob store + */ +public class RemoteTransientSettingsMetadata extends AbstractRemoteBlobObject { public static final String TRANSIENT_SETTING_METADATA = "transient-settings"; @@ -37,22 +39,16 @@ public class RemoteTransientSettingsMetadata extends AbstractRemoteBlobStoreObje private Settings transientSettings; private long metadataVersion; - private String blobName; - private final String clusterUUID; - public RemoteTransientSettingsMetadata(Settings transientSettings, long metadataVersion, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemoteTransientSettingsMetadata(Settings transientSettings, long metadataVersion, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.transientSettings = transientSettings; this.metadataVersion = metadataVersion; - this.clusterUUID = clusterUUID; } - public RemoteTransientSettingsMetadata(String blobName, String clusterUUID, BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + public RemoteTransientSettingsMetadata(String blobName, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); this.blobName = blobName; - this.clusterUUID = clusterUUID; } @Override @@ -60,11 +56,6 @@ public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of("global-metadata"), TRANSIENT_SETTING_METADATA); } - @Override - public String getFullBlobName() { - return blobName; - } - @Override public String generateBlobFileName() { String blobFileName = String.join( @@ -74,8 +65,7 @@ public String generateBlobFileName() { RemoteStoreUtils.invertLong(System.currentTimeMillis()), String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) ); - // setting the full blob path with name for future access - this.blobName = getBlobPathForUpload().buildAsString() + blobFileName; + this.blobFileName = blobFileName; return blobFileName; } @@ -84,15 +74,10 @@ public Settings get() { return transientSettings; } - @Override - public String clusterUUID() { - return clusterUUID; - } - - @Override public InputStream serialize() throws IOException { - return SETTINGS_METADATA_FORMAT.serialize(transientSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + return SETTINGS_METADATA_FORMAT.serialize(transientSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) + .streamInput(); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/package-info.java b/server/src/main/java/org/opensearch/gateway/remote/model/package-info.java new file mode 100644 index 0000000000000..c0d13d15cc885 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Package containing models for remote cluster state + */ +package org.opensearch.gateway.remote.model; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 0918236638016..a9b4b67d0e215 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -36,8 +36,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -46,7 +44,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import java.util.stream.Collectors; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; @@ -69,9 +66,9 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index a79205f174c68..4f0bd74a3c6c9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -36,7 +36,6 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; -import org.opensearch.common.util.concurrent.AbstractAsyncTask; import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesArray; @@ -45,8 +44,10 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; +import org.opensearch.gateway.remote.model.RemoteCustomMetadata; +import org.opensearch.gateway.remote.model.RemoteIndexMetadata; import org.opensearch.index.remote.RemoteIndexPathUploader; -import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; @@ -76,7 +77,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; @@ -95,7 +95,7 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java index 2cae382458c83..4c12ba92b7bcc 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java @@ -15,8 +15,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata.COMPONENT_PREFIX; -import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteIndexMetadata.INDEX_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; import java.io.IOException; import java.io.InputStream; @@ -31,6 +31,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.model.BlobPathParameters; +import org.opensearch.gateway.remote.model.RemoteIndexMetadata; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -70,41 +72,34 @@ public void tearDown() throws Exception { public void testGet() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); assertThat(remoteObjectForUpload.get(), is(indexMetadata)); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); assertThat(remoteObjectForDownload.get(), nullValue()); } public void testClusterUUID() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); } public void testFullBlobName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); } public void testBlobPathParameters() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertThat(params.getPathTokens(), is(List.of(INDEX_PATH_TOKEN))); assertThat(params.getFilePrefix(), is("metadata")); @@ -112,8 +107,7 @@ public void testBlobPathParameters() { public void testGenerateBlobFileName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); assertThat(nameTokens[0], is("metadata")); @@ -124,8 +118,7 @@ public void testGenerateBlobFileName() { public void testGetUploadedMetadata() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); try (InputStream inputStream = remoteObjectForUpload.serialize()) { @@ -137,8 +130,7 @@ public void testGetUploadedMetadata() throws IOException { public void testSerDe() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreTransferService, blobStoreRepository, - clusterName, threadPool); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); try (InputStream inputStream = remoteObjectForUpload.serialize()) { assertThat(inputStream.available(), greaterThan(0)); IndexMetadata readIndexMetadata = remoteObjectForUpload.deserialize(inputStream); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java index e0af7808a5853..d1a4b89fe4379 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java @@ -41,7 +41,7 @@ public class RemoteManifestManagerTests extends OpenSearchTestCase { public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); - remoteManifestManager = new RemoteManifestManager(blobStoreTransferService, blobStoreRepository, clusterSettings, "test-node-id", new TestThreadPool("test")); + remoteManifestManager = new RemoteManifestManager(blobStoreTransferService, blobStoreRepository, clusterSettings, "test-node-id", new TestThreadPool("test"), "test-cluster-name"); blobStoreTransferService = mock(BlobStoreTransferService.class); blobStore = mock(BlobStore.class); when(blobStoreRepository.blobStore()).thenReturn(blobStore); From 9b1cab6a5ac348f38127eb50a7d15ffafece3b64 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 29 May 2024 11:02:34 +0530 Subject: [PATCH 100/133] Add remote state implementation unit tests Signed-off-by: Sooraj Sinha --- .../model/AbstractRemoteBlobObject.java | 5 +- .../model/RemoteClusterMetadataManifest.java | 3 +- .../remote/model/RemoteClusterBlocksTest.java | 159 +++++++++++++++++ .../RemoteClusterMetadataManifestTests.java | 168 ++++++++++++++++++ .../{ => model}/RemoteIndexMetadataTests.java | 21 ++- 5 files changed, 350 insertions(+), 6 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java rename server/src/test/java/org/opensearch/gateway/remote/{ => model}/RemoteIndexMetadataTests.java (88%) diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java index 5ab621406bc85..b547d3584382a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java @@ -41,6 +41,9 @@ public String getFullBlobName() { public String getBlobFileName() { if (blobFileName == null) { + if (blobName == null) { + return null; + } String[] pathTokens = blobName.split(PATH_DELIMITER); blobFileName = pathTokens[pathTokens.length - 1]; } @@ -55,7 +58,7 @@ public String clusterUUID() { public abstract UploadedMetadata getUploadedMetadata(); - protected void setFullBlobName(BlobPath blobPath) { + public void setFullBlobName(BlobPath blobPath) { this.blobName = blobPath.buildAsString() + blobFileName; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index 5b4e10e005be2..170836ce57baf 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -79,7 +79,7 @@ public String generateBlobFileName() { // String blobFileName = String.join( DELIMITER, - MANIFEST_PATH_TOKEN, + MANIFEST_FILE_PREFIX, RemoteStoreUtils.invertLong(clusterMetadataManifest.getClusterTerm()), RemoteStoreUtils.invertLong(clusterMetadataManifest.getStateVersion()), (clusterMetadataManifest.isCommitted() ? "C" : "P"), // C for committed and P for published @@ -93,6 +93,7 @@ public String generateBlobFileName() { @Override public UploadedMetadata getUploadedMetadata() { + assert blobName != null; return new UploadedMetadataAttribute(MANIFEST_PATH_TOKEN, blobName); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java new file mode 100644 index 0000000000000..58c5058e95e74 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java @@ -0,0 +1,159 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; + +import java.io.IOException; +import java.io.InputStream; +import java.util.EnumSet; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.block.ClusterBlock; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteClusterBlocksTest extends OpenSearchTestCase { + + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long VERSION = 5L; + private static final String PATH_PARAM_TRANSIENT = "transient"; + + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testGet() { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.get(), is(clusterBlocks)); + + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.get(), nullValue()); + } + + public void testClusterUUID() { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathParameters() { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(PATH_PARAM_TRANSIENT))); + assertThat(params.getFilePrefix(), is(CLUSTER_BLOCKS)); + } + + public void testGenerateBlobFileName() { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertThat(nameTokens[0], is(CLUSTER_BLOCKS)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION))); + } + + public void testGetUploadedMetadata() throws IOException { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertThat(uploadedMetadata.getComponent(), is(CLUSTER_BLOCKS)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + ClusterBlocks clusterBlocks = getClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + assertThat(inputStream.available(), greaterThan(0)); + ClusterBlocks readClusterBlocks = remoteObjectForUpload.deserialize(inputStream); + assertThat(readClusterBlocks.global().size(), is(1)); + assertThat(readClusterBlocks.global().iterator().next(), is(clusterBlocks.global().iterator().next())); + } + } + + private ClusterBlocks getClusterBlocks() { + return new ClusterBlocks.Builder().addGlobalBlock(new ClusterBlock(12345, "test block", true, false, true, RestStatus.CREATED, EnumSet.of( + ClusterBlockLevel.WRITE))).build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java new file mode 100644 index 0000000000000..1e71c3c0dab0c --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java @@ -0,0 +1,168 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.opensearch.Version; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.ClusterStateDiffManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteClusterMetadataManifestTests extends OpenSearchTestCase { + + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + + private static final long TERM = 2L; + private static final long VERSION = 5L; + + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testGet() { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.get(), is(manifest)); + + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.get(), nullValue()); + } + + public void testClusterUUID() { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathParameters() { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(MANIFEST_PATH_TOKEN))); + assertThat(params.getFilePrefix(), is(MANIFEST_FILE_PREFIX)); + } + + public void testGenerateBlobFileName() { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertThat(nameTokens[0], is(MANIFEST_FILE_PREFIX)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(TERM)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), is(VERSION)); + assertThat(nameTokens[3], is("C")); + assertThat(RemoteStoreUtils.invertLong(nameTokens[4]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[5], is(String.valueOf(MANIFEST_CURRENT_CODEC_VERSION))); + } + + public void testGetUploadedMetadata() throws IOException { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertThat(uploadedMetadata.getComponent(), is(MANIFEST_PATH_TOKEN)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + ClusterMetadataManifest manifest = getClusterMetadataManifest(); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertThat(inputStream.available(), greaterThan(0)); + ClusterMetadataManifest readManifest = remoteObjectForUpload.deserialize(inputStream); + assertThat(readManifest, is(manifest)); + } + } + + private ClusterMetadataManifest getClusterMetadataManifest() { + return ClusterMetadataManifest.builder().opensearchVersion(Version.CURRENT).codecVersion(MANIFEST_CURRENT_CODEC_VERSION).nodeId("test-node") + .clusterUUID("test-uuid").previousClusterUUID("_NA_").stateUUID("state-uuid").clusterTerm(TERM).stateVersion(VERSION).committed(true) + .coordinationMetadata(new UploadedMetadataAttribute("test-attr", "uploaded-file")).diffManifest( + ClusterStateDiffManifest.builder().fromStateUUID("from-uuid").toStateUUID("to-uuid").indicesUpdated(Collections.emptyList()) + .indicesDeleted(Collections.emptyList()) + .customMetadataUpdated(Collections.emptyList()).customMetadataDeleted(Collections.emptyList()) + .indicesRoutingUpdated(Collections.emptyList()).indicesRoutingDeleted(Collections.emptyList()).build()).build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java similarity index 88% rename from server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java rename to server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java index 4c12ba92b7bcc..b6a4d0ba60ebe 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote; +package org.opensearch.gateway.remote.model; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; @@ -31,6 +31,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.Index; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.gateway.remote.model.BlobPathParameters; import org.opensearch.gateway.remote.model.RemoteIndexMetadata; import org.opensearch.index.remote.RemoteStoreUtils; @@ -42,7 +43,9 @@ public class RemoteIndexMetadataTests extends OpenSearchTestCase { - private static final String TEST_BLOB_NAME = "test-blob-name"; + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; private static final long VERSION = 5L; private String clusterUUID; @@ -97,11 +100,20 @@ public void testFullBlobName() { assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); } + public void testBlobFileName() { + IndexMetadata indexMetadata = getIndexMetadata(); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + public void testBlobPathParameters() { IndexMetadata indexMetadata = getIndexMetadata(); RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); - assertThat(params.getPathTokens(), is(List.of(INDEX_PATH_TOKEN))); + assertThat(params.getPathTokens(), is(List.of(INDEX_PATH_TOKEN, indexMetadata.getIndexUUID()))); assertThat(params.getFilePrefix(), is("metadata")); } @@ -121,7 +133,8 @@ public void testGetUploadedMetadata() throws IOException { RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); - try (InputStream inputStream = remoteObjectForUpload.serialize()) { + try (InputStream inputStream = remoteObjectForUpload.serialize()){ + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); assertThat(uploadedMetadata.getComponent(), is(COMPONENT_PREFIX + "test-index")); assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); From 9aeea2168642a8a2b61bbf79b3cef9c84ef354f5 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 29 May 2024 16:42:44 +0530 Subject: [PATCH 101/133] Update interface to use entity U Signed-off-by: Sooraj Sinha --- .../remote/model/AbstractRemoteBlobStore.java | 26 ++++++++----------- .../remote/model/RemoteObjectStore.java | 6 ++--- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java index d873fc4eed05a..55822a195bd1d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java @@ -28,7 +28,7 @@ * @param The entity which can be uploaded to / downloaded from blob store * @param The concrete class implementing {@link RemoteObject} which is used as a wrapper for T entity. */ -public class AbstractRemoteBlobStore> implements RemoteObjectStore { +public class AbstractRemoteBlobStore> implements RemoteObjectStore { private final BlobStoreTransferService transferService; private final BlobStoreRepository blobStoreRepository; @@ -44,31 +44,27 @@ public AbstractRemoteBlobStore(BlobStoreTransferService blobStoreTransferService } @Override - public void writeAsync(RemoteObject obj, ActionListener listener) { - assert obj instanceof AbstractRemoteBlobObject; - AbstractRemoteBlobObject blobStoreObj = (AbstractRemoteBlobObject) obj; - assert blobStoreObj.get() != null; + public void writeAsync(U obj, ActionListener listener) { + assert obj.get() != null; try { InputStream inputStream = obj.serialize(); - BlobPath blobPath = getBlobPathForUpload(blobStoreObj); - blobStoreObj.setFullBlobName(blobPath); - transferService.uploadBlob(inputStream, getBlobPathForUpload(blobStoreObj), blobStoreObj.getBlobFileName(), WritePriority.URGENT, listener); + BlobPath blobPath = getBlobPathForUpload(obj); + obj.setFullBlobName(blobPath); + transferService.uploadBlob(inputStream, getBlobPathForUpload(obj), obj.getBlobFileName(), WritePriority.URGENT, listener); } catch (Exception e) { listener.onFailure(e); } } @Override - public T read(RemoteObject obj) throws IOException { - assert obj instanceof AbstractRemoteBlobObject; - AbstractRemoteBlobObject blobStoreObj = (AbstractRemoteBlobObject) obj; - assert blobStoreObj.getFullBlobName() != null; - return blobStoreObj.deserialize( - transferService.downloadBlob(getBlobPathForDownload(blobStoreObj), blobStoreObj.getBlobFileName())); + public T read(U obj) throws IOException { + assert obj.getFullBlobName() != null; + return obj.deserialize( + transferService.downloadBlob(getBlobPathForDownload(obj), obj.getBlobFileName())); } @Override - public void readAsync(RemoteObject obj, ActionListener listener) { + public void readAsync(U obj, ActionListener listener) { executorService.execute(() -> { try { listener.onResponse(read(obj)); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java index b8ed8ebb7fa0c..92e494343b664 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java @@ -18,9 +18,9 @@ */ public interface RemoteObjectStore> { - public void writeAsync(RemoteObject obj, ActionListener listener); + public void writeAsync(U obj, ActionListener listener); - public T read(RemoteObject obj) throws IOException; + public T read(U obj) throws IOException; - public void readAsync(RemoteObject obj, ActionListener listener); + public void readAsync(U obj, ActionListener listener); } From 1952bacbe159553c57328c29a52d066734993d4d Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 30 May 2024 12:23:59 +0530 Subject: [PATCH 102/133] Remove duplicate tests Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateServiceIT.java | 115 ------------------ 1 file changed, 115 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index 283d8bb3913f0..4fb09797bd4d1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -10,7 +10,6 @@ import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest; import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse; -import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.settings.Settings; import org.opensearch.discovery.DiscoveryStats; @@ -23,14 +22,11 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; -import static org.opensearch.gateway.remote.RemoteClusterStateService.RETAINED_MANIFESTS; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; @@ -48,16 +44,6 @@ protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build(); } - private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { - internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); - internalCluster().startDataOnlyNodes(numDataOnlyNodes); - for (String index : indices.split(",")) { - createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); - ensureYellowAndNoInitializingShards(index); - ensureGreen(index); - } - } - private Map initialTestSetup(int shardCount, int replicaCount, int dataNodeCount, int clusterManagerNodeCount) { prepareCluster(clusterManagerNodeCount, dataNodeCount, INDEX_NAME, replicaCount, shardCount); Map indexStats = indexData(1, false, INDEX_NAME); @@ -66,107 +52,6 @@ private Map initialTestSetup(int shardCount, int replicaCount, int return indexStats; } - public void testRemoteCleanupTaskUpdated() { - int shardCount = randomIntBetween(1, 2); - int replicaCount = 1; - int dataNodeCount = shardCount * (replicaCount + 1); - int clusterManagerNodeCount = 1; - - initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); - RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( - RemoteClusterStateService.class - ); - - assertEquals(5, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); - assertTrue(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); - - // now disable - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), -1)) - .get(); - - assertEquals(-1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMillis()); - assertFalse(remoteClusterStateService.getStaleFileDeletionTask().isScheduled()); - - // now set Clean up interval to 1 min - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) - .get(); - assertEquals(1, remoteClusterStateService.getStaleFileDeletionTask().getInterval().getMinutes()); - } - - public void testRemoteCleanupOnlyAfter10Updates() throws Exception { - int shardCount = randomIntBetween(1, 2); - int replicaCount = 1; - int dataNodeCount = shardCount * (replicaCount + 1); - int clusterManagerNodeCount = 1; - - initialTestSetup(shardCount, replicaCount, dataNodeCount, clusterManagerNodeCount); - RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance( - RemoteClusterStateService.class - ); - - // set cleanup interval to 1 min - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings(Settings.builder().put(REMOTE_CLUSTER_STATE_CLEANUP_INTERVAL_SETTING.getKey(), "1m")) - .get(); - - replicaCount = updateReplicaCountNTimes(9, replicaCount); - - RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class); - - BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME); - BlobPath baseMetadataPath = repository.basePath() - .add( - Base64.getUrlEncoder() - .withoutPadding() - .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8)) - ) - .add("cluster-state") - .add(getClusterState().metadata().clusterUUID()); - BlobPath manifestContainerPath = baseMetadataPath.add("manifest"); - - assertBusy(() -> { - assertEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); - }, 1, TimeUnit.MINUTES); - - replicaCount = updateReplicaCountNTimes(8, replicaCount); - - // wait for 1 min, to ensure that clean up task ran and didn't clean up stale files because it was less than 10 - Thread.sleep(60000); - assertNotEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); - - // Do 2 more updates, now since the total successful state changes are more than 10, stale files will be cleaned up - replicaCount = updateReplicaCountNTimes(2, replicaCount); - - assertBusy(() -> { - assertEquals( - RETAINED_MANIFESTS - 1, - repository.blobStore().blobContainer(manifestContainerPath).listBlobsByPrefix("manifest").size() - ); - }, 1, TimeUnit.MINUTES); - - Map indexMetadataMap = remoteClusterStateService.getLatestClusterState( - cluster().getClusterName(), - getClusterState().metadata().clusterUUID(), - false - ).getMetadata().getIndices(); - assertEquals(replicaCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfReplicas()); - assertEquals(shardCount, indexMetadataMap.values().stream().findFirst().get().getNumberOfShards()); - } - public void testRemoteStateStats() { int shardCount = randomIntBetween(1, 2); int replicaCount = 1; From fc9160c72185289b639f7886fc0f4e7225b54227 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 30 May 2024 12:24:42 +0530 Subject: [PATCH 103/133] Add parsing for DiffableStringMap Signed-off-by: Shivansh Arora --- .../cluster/metadata/DiffableStringMap.java | 32 +++++++++++- .../metadata/DiffableStringMapTests.java | 51 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java index a8102182576ff..c2f13fe720dc4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java @@ -37,6 +37,9 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.AbstractMap; @@ -46,6 +49,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.core.xcontent.XContentParser.Token.END_OBJECT; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; /** * This is a {@code Map} that implements AbstractDiffable so it @@ -54,7 +62,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class DiffableStringMap extends AbstractMap implements Diffable { +public class DiffableStringMap extends AbstractMap implements Diffable, ToXContentObject { public static final DiffableStringMap EMPTY = new DiffableStringMap(Collections.emptyMap()); @@ -90,6 +98,28 @@ public static Diff readDiffFrom(StreamInput in) throws IOExce return new DiffableStringMapDiff(in); } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("inner_map", innerMap); + builder.endObject(); + return builder; + } + + public static DiffableStringMap fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + ensureFieldName(parser, parser.currentToken(), "inner_map"); + parser.nextToken(); + Map innerMap = parser.mapOrdered(); + ensureExpectedToken(END_OBJECT, parser.currentToken(), parser); + return new DiffableStringMap(innerMap.entrySet().stream().collect(Collectors.toMap(Entry::getKey, entry -> (String) entry.getValue()))); + } + /** * Represents differences between two DiffableStringMaps. * diff --git a/server/src/test/java/org/opensearch/cluster/metadata/DiffableStringMapTests.java b/server/src/test/java/org/opensearch/cluster/metadata/DiffableStringMapTests.java index fcf6ad97c0f00..8fc1953e69663 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/DiffableStringMapTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/DiffableStringMapTests.java @@ -34,11 +34,21 @@ import org.opensearch.cluster.Diff; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.suggest.phrase.Correction; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -113,4 +123,45 @@ public void testSerialization() throws IOException { DiffableStringMap deserialized = DiffableStringMap.readFrom(bso.bytes().streamInput()); assertThat(deserialized, equalTo(dsm)); } + + public void testToXContent() throws IOException { + Map m = new HashMap<>(); + if (frequently()) { + m.put("foo", "bar"); + m.put("baz", "eggplant"); + m.put("potato", "canon"); + } + DiffableStringMap dsm = new DiffableStringMap(m); + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + dsm.toXContent(builder, null); + String expectedString = "{\n" + + " \"inner_map\" : {\n" + + m.entrySet().stream().map(entry -> " \"" + entry.getKey() + "\" : \"" + entry.getValue() + "\"").collect(Collectors.joining(",\n")) + "\n" + + " }\n" + + "}"; + assertEquals(expectedString, builder.toString()); + } + + public void testFromXContent() throws IOException { + Map m = new HashMap<>(); + if (frequently()) { + m.put("foo", "bar"); + m.put("baz", "eggplant"); + m.put("potato", "canon"); + } + DiffableStringMap dsm = new DiffableStringMap(m); + boolean humanReadable = randomBoolean(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + dsm, + mediaType, + null, + humanReadable + ); + + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + DiffableStringMap parsed = DiffableStringMap.fromXContent(parser); + assertEquals(dsm, parsed); + } + } } From 35970d98ffeb075fdda5c0d460e03896009f6e6f Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 30 May 2024 16:35:06 +0530 Subject: [PATCH 104/133] Add status to cluster block serialization Signed-off-by: Shivansh Arora --- .../cluster/block/ClusterBlock.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java index e131e4facc4b3..c958d7752cdbb 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlock.java @@ -32,6 +32,7 @@ package org.opensearch.cluster.block; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.util.set.Sets; @@ -49,6 +50,9 @@ import java.util.Objects; import java.util.Set; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_API; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_PARAM; + /** * Blocks the cluster for concurrency * @@ -62,6 +66,8 @@ public class ClusterBlock implements Writeable, ToXContentFragment { static final String KEY_RETRYABLE = "retryable"; static final String KEY_DISABLE_STATE_PERSISTENCE = "disable_state_persistence"; static final String KEY_LEVELS = "levels"; + static final String KEY_STATUS = "status"; + static final String KEY_ALLOW_RELEASE_RESOURCES = "allow_release_resources"; private static final Set VALID_FIELDS = Sets.newHashSet( KEY_UUID, KEY_DESCRIPTION, @@ -169,6 +175,7 @@ public boolean disableStatePersistence() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Metadata.XContentContext context = Metadata.XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, CONTEXT_MODE_API)); builder.startObject(Integer.toString(id)); if (uuid != null) { builder.field(KEY_UUID, uuid); @@ -183,6 +190,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.value(level.name().toLowerCase(Locale.ROOT)); } builder.endArray(); + if (context == Metadata.XContentContext.GATEWAY) { + builder.field(KEY_STATUS, status); + builder.field(KEY_ALLOW_RELEASE_RESOURCES, allowReleaseResources); + } builder.endObject(); return builder; } @@ -192,6 +203,8 @@ public static ClusterBlock fromXContent(XContentParser parser, int id) throws IO String description = null; boolean retryable = false; boolean disableStatePersistence = false; + RestStatus status = null; + boolean allowReleaseResources = false; EnumSet levels = EnumSet.noneOf(ClusterBlockLevel.class); String currentFieldName = skipBlockID(parser); XContentParser.Token token; @@ -212,6 +225,12 @@ public static ClusterBlock fromXContent(XContentParser parser, int id) throws IO case KEY_DISABLE_STATE_PERSISTENCE: disableStatePersistence = parser.booleanValue(); break; + case KEY_STATUS: + status = RestStatus.valueOf(parser.text()); + break; + case KEY_ALLOW_RELEASE_RESOURCES: + allowReleaseResources = parser.booleanValue(); + break; default: throw new IllegalArgumentException("unknown field [" + currentFieldName + "]"); } @@ -227,7 +246,7 @@ public static ClusterBlock fromXContent(XContentParser parser, int id) throws IO throw new IllegalArgumentException("unexpected token [" + token + "]"); } } - return new ClusterBlock(id, uuid, description, retryable, disableStatePersistence, false, null, levels); + return new ClusterBlock(id, uuid, description, retryable, disableStatePersistence, allowReleaseResources, status, levels); } private static String skipBlockID(XContentParser parser) throws IOException { From e5413185a39aebc6ff7dd03b440f751e5e6cfe06 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Thu, 30 May 2024 16:44:30 +0530 Subject: [PATCH 105/133] fix cluster block tests Signed-off-by: Shivansh Arora --- .../cluster/block/ClusterBlockTests.java | 27 ++++++++++++++++--- .../cluster/block/ClusterBlocksTests.java | 4 +-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java index 914a778e379c9..7800873a5591c 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java @@ -33,6 +33,7 @@ package org.opensearch.cluster.block; import org.opensearch.Version; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.json.JsonXContent; @@ -53,7 +54,10 @@ import java.util.Locale; import java.util.Map; +import static java.util.Collections.singletonMap; import static java.util.EnumSet.copyOf; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_API; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_GATEWAY; import static org.opensearch.test.VersionUtils.randomVersion; import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.equalTo; @@ -145,14 +149,26 @@ public void testGetIndexBlockWithId() { assertThat(builder.build().getIndexBlockWithId("index", randomValueOtherThan(blockId, OpenSearchTestCase::randomInt)), nullValue()); } - public void testToXContent() throws IOException { + public void testToXContent_APIMode() throws IOException { ClusterBlock clusterBlock = randomClusterBlock(); XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); builder.startObject(); clusterBlock.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); - String expectedString = "{\n" + getExpectedXContentFragment(clusterBlock, " ") + "\n}"; + String expectedString = "{\n" + getExpectedXContentFragment(clusterBlock, " ", false) + "\n}"; + + assertEquals(expectedString, builder.toString()); + } + + public void testToXContent_GatewayMode() throws IOException { + ClusterBlock clusterBlock = randomClusterBlock(); + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + builder.startObject(); + clusterBlock.toXContent(builder, new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY))); + builder.endObject(); + + String expectedString = "{\n" + getExpectedXContentFragment(clusterBlock, " ", true) + "\n}"; assertEquals(expectedString, builder.toString()); } @@ -197,7 +213,7 @@ private void doFromXContentTestWithRandomFields(boolean addRandomFields) throws } } - static String getExpectedXContentFragment(ClusterBlock clusterBlock, String indent) { + static String getExpectedXContentFragment(ClusterBlock clusterBlock, String indent, boolean gatewayMode) { return indent + "\"" + clusterBlock.id() @@ -216,7 +232,7 @@ static String getExpectedXContentFragment(ClusterBlock clusterBlock, String inde : "") + String.format( Locale.ROOT, - indent + " \"levels\" : [%s]\n", + indent + " \"levels\" : [%s]", clusterBlock.levels().isEmpty() ? " " : "\n" @@ -230,6 +246,9 @@ static String getExpectedXContentFragment(ClusterBlock clusterBlock, String inde + "\n " + indent ) + + (gatewayMode ? ",\n" + + indent + " \"status\" : \"" + clusterBlock.status() + "\",\n" + + indent + " \"allow_release_resources\" : " + clusterBlock.isAllowReleaseResources() + "\n" : "\n" ) + indent + "}"; } diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java index 86dda9eaa1a04..c1d4401760be3 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlocksTests.java @@ -48,7 +48,7 @@ public void testToXContent() throws IOException { : " \"global\" : {\n" + clusterBlocks.global() .stream() - .map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")) + .map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ", false)) .collect(Collectors.joining(",\n")) + "\n }" + (!clusterBlocks.indices().isEmpty() ? "," : "") @@ -72,7 +72,7 @@ public void testToXContent() throws IOException { : "\n" + entry.getValue() .stream() - .map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ")) + .map(clusterBlock -> getExpectedXContentFragment(clusterBlock, " ", false)) .collect(Collectors.joining(",\n")) + "\n }") ) From e8fec0381cbab3fedd4adae866a0b44fe5acf336 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Thu, 30 May 2024 18:50:36 +0530 Subject: [PATCH 106/133] Minor refactoring for read flow Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 38 +++++++++++++++++++ .../remote/ClusterStateDiffManifest.java | 5 ++- .../remote/RemoteClusterStateService.java | 6 +-- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 5d105db4238d5..046b56191d9f1 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -14,6 +14,7 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; import org.opensearch.common.blobstore.BlobContainer; @@ -49,6 +50,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.function.Function; @@ -264,6 +266,42 @@ public IndexRoutingTableInputStreamReader read(BlobContainer blobContainer, Stri return null; } + public List getUpdatedIndexRoutingTableMetadata(List updatedIndicesRouting, List allIndicesRouting) { + return updatedIndicesRouting.stream().map(idx -> { + Optional uploadedIndexMetadataOptional = allIndicesRouting.stream().filter(idx2 -> idx2.getIndexName().equals(idx)).findFirst(); + assert uploadedIndexMetadataOptional.isPresent() == true; + return uploadedIndexMetadataOptional.get(); + }).collect(Collectors.toList()); + } + + public static List getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List deletedIndicesRouting = new ArrayList<>(); + for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { + if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is deleted + deletedIndicesRouting.add(previousIndexRouting.getIndex().getName()); + } + } + return deletedIndicesRouting; + } + + public static List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List updatedIndicesRouting = new ArrayList<>(); + for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { + if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is created + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } else { + if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { + // if the latest routing table has the same routing table as the previous routing table, then the index is not updated + continue; + } + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } + } + return updatedIndicesRouting; + } + @Override public void close() throws IOException { if (blobStoreRepository != null) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index a7dc3358766e5..b1eb80821b10c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; @@ -83,8 +84,8 @@ public class ClusterStateDiffManifest implements ToXContentObject { customMetadataDeleted.add(custom); } } - indicesRoutingUpdated = getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); - indicesRoutingDeleted = getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); + indicesRoutingUpdated = RemoteRoutingTableService.getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); + indicesRoutingDeleted = RemoteRoutingTableService.getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); } public ClusterStateDiffManifest(String fromStateUUID, diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 23e4b07b3ee28..2ff634f22e9cd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -1106,11 +1106,7 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata return uploadedIndexMetadataOptional.get(); }).collect(Collectors.toList()); - List updatedIndexRouting = diff.getIndicesRoutingUpdated().stream().map(idx -> { - Optional uploadedIndexMetadataOptional = manifest.getIndicesRouting().stream().filter(idx2 -> idx2.getIndexName().equals(idx)).findFirst(); - assert uploadedIndexMetadataOptional.isPresent() == true; - return uploadedIndexMetadataOptional.get(); - }).collect(Collectors.toList()); + List updatedIndexRouting = remoteRoutingTableService.getUpdatedIndexRoutingTableMetadata(diff.getIndicesRoutingUpdated(), manifest.getIndicesRouting()); Map updatedCustomMetadata = new HashMap<>(); if (diff.getCustomMetadataUpdated() != null) { From 00f0249ccf08c0bdf1712643c673af8950d61d23 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 31 May 2024 14:20:05 +0530 Subject: [PATCH 107/133] Upload consistent settings and customs object to remote Signed-off-by: Shivansh Arora --- .../org/opensearch/cluster/ClusterModule.java | 29 ++++ .../org/opensearch/cluster/ClusterState.java | 4 + .../cluster/metadata/DiffableStringMap.java | 10 +- .../opensearch/cluster/metadata/Metadata.java | 5 + .../remote/ClusterMetadataManifest.java | 101 +++++++++++-- .../remote/ClusterStateDiffManifest.java | 112 ++++++++++++-- .../RemoteClusterStateAttributesManager.java | 44 +++++- .../remote/RemoteClusterStateService.java | 141 +++++++++++++----- .../remote/RemoteClusterStateUtils.java | 12 +- .../remote/RemoteGlobalMetadataManager.java | 19 ++- .../gateway/remote/RemoteManifestManager.java | 29 ++-- .../model/RemoteClusterStateCustoms.java | 99 ++++++++++++ .../RemoteClusterStateCustomsBlobStore.java | 20 +++ .../remote/model/RemoteCustomMetadata.java | 3 +- .../remote/model/RemoteDiscoveryNodes.java | 5 +- .../RemoteHashesOfConsistentSettings.java | 88 +++++++++++ ...teHashesOfConsistentSettingsBlobStore.java | 20 +++ ...RemoteClusterStateCleanupManagerTests.java | 2 +- 18 files changed, 652 insertions(+), 91 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsBlobStore.java diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index aa9101090b6d5..1e389da479f44 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -318,6 +318,35 @@ public static List getNamedXWriteables() { DecommissionAttributeMetadata::fromXContent ) ); + // Cluster State + entries.add( + new NamedXContentRegistry.Entry( + ClusterState.Custom.class, + new ParseField(SnapshotsInProgress.TYPE), + SnapshotsInProgress::fromXContent + ) + ); + entries.add( + new NamedXContentRegistry.Entry( + ClusterState.Custom.class, + new ParseField(RestoreInProgress.TYPE), + RestoreInProgress::fromXContent + ) + ); + entries.add( + new NamedXContentRegistry.Entry( + ClusterState.Custom.class, + new ParseField(SnapshotDeletionsInProgress.TYPE), + SnapshotDeletionsInProgress::fromXContent + ) + ); + entries.add( + new NamedXContentRegistry.Entry( + ClusterState.Custom.class, + new ParseField(RepositoryCleanupInProgress.TYPE), + RepositoryCleanupInProgress::fromXContent + ) + ); return entries; } diff --git a/server/src/main/java/org/opensearch/cluster/ClusterState.java b/server/src/main/java/org/opensearch/cluster/ClusterState.java index e38936d3e2622..ef3695b8d8c20 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterState.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterState.java @@ -61,6 +61,7 @@ import org.opensearch.core.common.io.stream.VersionedNamedWriteable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.discovery.Discovery; import java.io.IOException; @@ -154,6 +155,9 @@ default boolean isPrivate() { return false; } + static Custom fromXContent(XContentParser parser, String name) throws IOException { + return parser.namedObject(Custom.class, name, null); + } } private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java index c2f13fe720dc4..57f3051b96226 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java @@ -74,7 +74,7 @@ public static DiffableStringMap readFrom(StreamInput in) throws IOException { return map.isEmpty() ? EMPTY : new DiffableStringMap(map); } - DiffableStringMap(final Map map) { + public DiffableStringMap(final Map map) { this.innerMap = Collections.unmodifiableMap(map); } @@ -100,7 +100,7 @@ public static Diff readDiffFrom(StreamInput in) throws IOExce @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); + builder.startObject("diffable_string_map"); builder.field("inner_map", innerMap); builder.endObject(); return builder; @@ -110,9 +110,9 @@ public static DiffableStringMap fromXContent(XContentParser parser) throws IOExc if (parser.currentToken() == null) { // fresh parser? move to next token parser.nextToken(); } - if (parser.currentToken() == XContentParser.Token.START_OBJECT) { - parser.nextToken(); - } + ensureFieldName(parser, parser.currentToken(), "diffable_string_map"); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + parser.nextToken(); ensureFieldName(parser, parser.currentToken(), "inner_map"); parser.nextToken(); Map innerMap = parser.mapOrdered(); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 14836371c7887..ad83b5b801193 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -986,6 +986,11 @@ public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata meta return metadata1.templates.equals(metadata2.templates); } + public static boolean isHashesOfConsistentSettingsEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings); + } + + public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) { int customCount1 = 0; for (Map.Entry cursor : metadata1.customs.entrySet()) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 863652c8492d6..92bec556afe97 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -65,6 +65,8 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata"); private static final ParseField UPLOADED_DISCOVERY_NODES_METADATA = new ParseField("uploaded_discovery_nodes_metadata"); private static final ParseField UPLOADED_CLUSTER_BLOCKS_METADATA = new ParseField("uploaded_cluster_blocks_metadata"); + private static final ParseField UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA = new ParseField("uploaded_hashes_of_consistent_settings_metadata"); + private static final ParseField UPLOADED_CLUSTER_STATE_CUSTOM_METADATA = new ParseField("uploaded_cluster_state_custom_metadata"); private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest"); private static final ParseField ROUTING_TABLE_VERSION_FIELD = new ParseField("routing_table_version"); private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); @@ -104,7 +106,9 @@ private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields .routingTableVersion(routingTableVersion(fields)) .indicesRouting(indicesRouting(fields)) .metadataVersion(metadataVersion(fields)) - .transientSettingsMetadata(transientSettingsMetadata(fields)); + .transientSettingsMetadata(transientSettingsMetadata(fields)) + .hashesOfConsistentSettings(hashesOfConsistentSettings(fields)) + .clusterStateCustomMetadataMap(clusterStateCustomMetadata(fields)); } private static long term(Object[] fields) { @@ -200,6 +204,15 @@ private static UploadedMetadataAttribute transientSettingsMetadata(Object[] fiel return (UploadedMetadataAttribute) fields[21]; } + private static UploadedMetadataAttribute hashesOfConsistentSettings(Object[] fields) { + return (UploadedMetadataAttribute) fields[22]; + } + + private static Map clusterStateCustomMetadata(Object[] fields) { + List customs = (List) fields[23]; + return customs.stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())); + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> manifestV0Builder(fields).build() @@ -299,6 +312,16 @@ private static void declareParser(ConstructingObjectParser indicesRouting; + private final UploadedMetadataAttribute uploadedHashesOfConsistentSettings; + private final Map uploadedClusterStateCustomMap; public List getIndices() { return indices; @@ -412,13 +437,26 @@ public Map getCustomMetadataMap() { return uploadedCustomMetadataMap; } - public long getRoutingTableVersion() { - return routingTableVersion; - } + public UploadedMetadataAttribute getDiscoverNodeMetadata() { + return uploadedDiscoveryNodesMetadata; + } + + public Map getClusterStateCustomMap() { + return uploadedClusterStateCustomMap; + } + + public UploadedMetadataAttribute getHashesOfConsistentSettings() { + return uploadedHashesOfConsistentSettings; + } + + public long getRoutingTableVersion() { + return routingTableVersion; + } + + public List getIndicesRouting() { + return indicesRouting; + } - public List getIndicesRouting() { - return indicesRouting; - } public boolean hasMetadataAttributesFiles() { return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null @@ -449,7 +487,9 @@ public ClusterMetadataManifest( long routingTableVersion, List indicesRouting, long metadataVersion, - UploadedMetadataAttribute uploadedTransientSettingsMetadata + UploadedMetadataAttribute uploadedTransientSettingsMetadata, + UploadedMetadataAttribute uploadedHashesOfConsistentSettings, + Map uploadedClusterStateCustomMap ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -476,6 +516,10 @@ public ClusterMetadataManifest( this.indicesRouting = Collections.unmodifiableList(indicesRouting); this.metadataVersion = metadataVersion; this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata; + this.uploadedHashesOfConsistentSettings = uploadedHashesOfConsistentSettings; + this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( + uploadedClusterStateCustomMap != null ? uploadedClusterStateCustomMap : new HashMap<>() + ); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -507,6 +551,10 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.metadataVersion = in.readLong(); this.uploadedTransientSettingsMetadata = new UploadedMetadataAttribute(in); + this.uploadedHashesOfConsistentSettings = new UploadedMetadataAttribute(in); + this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( + in.readMap(StreamInput::readString, UploadedMetadataAttribute::new) + ); } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); this.globalMetadataFileName = in.readString(); @@ -521,6 +569,8 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.diffManifest = null; this.metadataVersion = -1; this.uploadedTransientSettingsMetadata = null; + this.uploadedHashesOfConsistentSettings = null; + this.uploadedClusterStateCustomMap = null; } else { this.codecVersion = CODEC_V0; // Default codec this.globalMetadataFileName = null; @@ -535,6 +585,8 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.diffManifest = null; this.metadataVersion = -1; this.uploadedTransientSettingsMetadata = null; + this.uploadedHashesOfConsistentSettings = null; + this.uploadedClusterStateCustomMap = null; } } @@ -625,6 +677,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endArray(); + builder.startObject(UPLOADED_CLUSTER_STATE_CUSTOM_METADATA.getPreferredName()); + for (UploadedMetadataAttribute attribute : getClusterStateCustomMap().values()) { + attribute.toXContent(builder, params); + } + builder.endObject(); + if (getHashesOfConsistentSettings() != null) { + builder.startObject(UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA.getPreferredName()); + getHashesOfConsistentSettings().toXContent(builder, params); + builder.endObject(); + } } return builder; } @@ -651,6 +713,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(indicesRouting); out.writeLong(metadataVersion); uploadedTransientSettingsMetadata.writeTo(out); + uploadedHashesOfConsistentSettings.writeTo(out); + out.writeMap(uploadedClusterStateCustomMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); @@ -659,6 +723,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public boolean equals(Object o) { + // ToDo: update this method to contain new attributes if (this == o) { return true; } @@ -684,6 +749,7 @@ public boolean equals(Object o) { @Override public int hashCode() { + // ToDo: update this method to contain new attributes return Objects.hash( codecVersion, globalMetadataFileName, @@ -753,6 +819,8 @@ public static class Builder { private ClusterStateDiffManifest diffManifest; private long routingTableVersion; private List indicesRouting; + private UploadedMetadataAttribute hashesOfConsistentSettings; + private Map clusterStateCustomMetadataMap; public Builder indices(List indices) { this.indices = indices; @@ -882,10 +950,21 @@ public Builder diffManifest(ClusterStateDiffManifest diffManifest) { return this; } + public Builder hashesOfConsistentSettings(UploadedMetadataAttribute hashesOfConsistentSettings) { + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + return this; + } + + public Builder clusterStateCustomMetadataMap(Map clusterStateCustomMetadataMap) { + this.clusterStateCustomMetadataMap = clusterStateCustomMetadataMap; + return this; + } + public Builder() { indices = new ArrayList<>(); customMetadataMap = new HashMap<>(); indicesRouting = new ArrayList<>(); + clusterStateCustomMetadataMap = new HashMap<>(); } public Builder(ClusterMetadataManifest manifest) { @@ -908,6 +987,8 @@ public Builder(ClusterMetadataManifest manifest) { this.diffManifest = manifest.diffManifest; this.routingTableVersion = manifest.routingTableVersion; this.indicesRouting = new ArrayList<>(manifest.indicesRouting); + this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings; + this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap; } public ClusterMetadataManifest build() { @@ -934,7 +1015,9 @@ public ClusterMetadataManifest build() { routingTableVersion, indicesRouting, metadataVersion, - transientSettingsMetadata + transientSettingsMetadata, + hashesOfConsistentSettings, + clusterStateCustomMetadataMap ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index b1eb80821b10c..2ad61c8da2a5f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -39,6 +39,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff"; private static final String TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD = "transient_settings_metadata_diff"; private static final String TEMPLATES_METADATA_UPDATED_FIELD = "templates_metadata_diff"; + private static final String HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD = "hashes_of_consistent_settings_diff"; private static final String INDICES_DIFF_FIELD = "indices_diff"; private static final String METADATA_CUSTOM_DIFF_FIELD = "metadata_custom_diff"; private static final String UPSERTS_FIELD = "upserts"; @@ -46,6 +47,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { private static final String CLUSTER_BLOCKS_UPDATED_FIELD = "cluster_blocks_diff"; private static final String DISCOVERY_NODES_UPDATED_FIELD = "discovery_nodes_diff"; private static final String ROUTING_TABLE_DIFF = "routing_table_diff"; + private static final String CLUSTER_STATE_CUSTOM_DIFF_FIELD = "cluster_state_custom_diff"; private final String fromStateUUID; private final String toStateUUID; private final boolean coordinationMetadataUpdated; @@ -60,6 +62,9 @@ public class ClusterStateDiffManifest implements ToXContentObject { private final boolean discoveryNodesUpdated; private final List indicesRoutingUpdated; private final List indicesRoutingDeleted; + private final boolean hashesOfConsistentSettingsUpdated; + private final List clusterStateCustomUpdated; + private final List clusterStateCustomDeleted; ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { fromStateUUID = previousState.stateUUID(); @@ -74,7 +79,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); customMetadataUpdated = new ArrayList<>(); for (String custom : state.metadata().customs().keySet()) { - if (!state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) { + if (!previousState.metadata().customs().containsKey(custom) || !state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) { customMetadataUpdated.add(custom); } } @@ -86,22 +91,40 @@ public class ClusterStateDiffManifest implements ToXContentObject { } indicesRoutingUpdated = RemoteRoutingTableService.getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); indicesRoutingDeleted = RemoteRoutingTableService.getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); + hashesOfConsistentSettingsUpdated = !state.metadata().hashesOfConsistentSettings().equals(previousState.metadata().hashesOfConsistentSettings()); + clusterStateCustomUpdated = new ArrayList<>(); + clusterStateCustomDeleted = new ArrayList<>(); + for (String custom : state.customs().keySet()) { + if (!previousState.customs().containsKey(custom) || !state.customs().get(custom).equals(previousState.customs().get(custom))) { + clusterStateCustomUpdated.add(custom); + } + } + for (String custom : previousState.customs().keySet()) { + if (state.customs().get(custom) == null) { + clusterStateCustomDeleted.add(custom); + } + } } - public ClusterStateDiffManifest(String fromStateUUID, - String toStateUUID, - boolean coordinationMetadataUpdated, - boolean settingsMetadataUpdated, - boolean transientSettingsMetadataUpdate, - boolean templatesMetadataUpdated, - List customMetadataUpdated, - List customMetadataDeleted, - List indicesUpdated, - List indicesDeleted, - boolean clusterBlocksUpdated, - boolean discoveryNodesUpdated, - ListindicesRoutingUpdated, - ListindicesRoutingDeleted) { + public ClusterStateDiffManifest( + String fromStateUUID, + String toStateUUID, + boolean coordinationMetadataUpdated, + boolean settingsMetadataUpdated, + boolean transientSettingsMetadataUpdate, + boolean templatesMetadataUpdated, + List customMetadataUpdated, + List customMetadataDeleted, + List indicesUpdated, + List indicesDeleted, + boolean clusterBlocksUpdated, + boolean discoveryNodesUpdated, + ListindicesRoutingUpdated, + ListindicesRoutingDeleted, + boolean hashesOfConsistentSettingsUpdated, + List clusterStateCustomUpdated, + List clusterStateCustomDeleted + ) { this.fromStateUUID = fromStateUUID; this.toStateUUID = toStateUUID; this.coordinationMetadataUpdated = coordinationMetadataUpdated; @@ -116,6 +139,9 @@ public ClusterStateDiffManifest(String fromStateUUID, this.discoveryNodesUpdated = discoveryNodesUpdated; this.indicesRoutingUpdated = indicesRoutingUpdated; this.indicesRoutingDeleted = indicesRoutingDeleted; + this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated; + this.clusterStateCustomUpdated = clusterStateCustomUpdated; + this.clusterStateCustomDeleted = clusterStateCustomDeleted; } @Override @@ -153,6 +179,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); builder.endObject(); + builder.field(HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD, hashesOfConsistentSettingsUpdated); } builder.endObject(); builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); @@ -170,6 +197,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); builder.endObject(); + builder.startObject(CLUSTER_STATE_CUSTOM_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String custom : clusterStateCustomUpdated) { + builder.value(custom); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String custom : clusterStateCustomDeleted) { + builder.value(custom); + } + builder.endArray(); + builder.endObject(); + } return builder; } @@ -207,6 +247,9 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw case TEMPLATES_METADATA_UPDATED_FIELD: builder.templatesMetadataUpdated(parser.booleanValue()); break; + case HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD: + builder.hashesOfConsistentSettingsUpdated(parser.booleanValue()); + break; default: throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); } @@ -263,6 +306,21 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); } } + } else if (currentFieldName.equals(CLUSTER_STATE_CUSTOM_DIFF_FIELD)) { + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.clusterStateCustomUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.clusterStateCustomDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } } else { throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); } @@ -421,6 +479,10 @@ public static class Builder { private boolean discoveryNodesUpdated; private List indicesRoutingUpdated; private List indicesRoutingDeleted; + private boolean hashesOfConsistentSettingsUpdated; + private List clusterStateCustomUpdated; + private List clusterStateCustomDeleted; + public Builder() {} public Builder fromStateUUID(String fromStateUUID) { @@ -453,6 +515,11 @@ public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { return this; } + public Builder hashesOfConsistentSettingsUpdated(boolean hashesOfConsistentSettingsUpdated) { + this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated; + return this; + } + public Builder customMetadataUpdated(List customMetadataUpdated) { this.customMetadataUpdated = customMetadataUpdated; return this; @@ -493,6 +560,16 @@ public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { return this; } + public Builder clusterStateCustomUpdated(List clusterStateCustomUpdated) { + this.clusterStateCustomUpdated = clusterStateCustomUpdated; + return this; + } + + public Builder clusterStateCustomDeleted(List clusterStateCustomDeleted) { + this.clusterStateCustomDeleted = clusterStateCustomDeleted; + return this; + } + public ClusterStateDiffManifest build() { return new ClusterStateDiffManifest( fromStateUUID, @@ -508,7 +585,10 @@ public ClusterStateDiffManifest build() { clusterBlocksUpdated, discoveryNodesUpdated, indicesRoutingUpdated, - indicesRoutingDeleted + indicesRoutingDeleted, + hashesOfConsistentSettingsUpdated, + clusterStateCustomUpdated, + clusterStateCustomDeleted ); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 1130b92289cc2..99e48d44b83ca 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -11,6 +11,7 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.CheckedRunnable; import org.opensearch.core.action.ActionListener; @@ -19,14 +20,25 @@ import org.opensearch.gateway.remote.model.AbstractRemoteBlobObject; import org.opensearch.gateway.remote.model.RemoteClusterBlocks; import org.opensearch.gateway.remote.model.RemoteClusterBlocksBlobStore; +import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms; +import org.opensearch.gateway.remote.model.RemoteClusterStateCustomsBlobStore; import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes; import org.opensearch.gateway.remote.model.RemoteDiscoveryNodesBlobStore; +import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings; +import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettingsBlobStore; import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; +import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; public class RemoteClusterStateAttributesManager { public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; @@ -39,6 +51,7 @@ public class RemoteClusterStateAttributesManager { private final String clusterName; private final RemoteClusterBlocksBlobStore clusterBlocksBlobStore; private final RemoteDiscoveryNodesBlobStore discoveryNodesBlobStore; + private final RemoteClusterStateCustomsBlobStore customsBlobStore; RemoteClusterStateAttributesManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository repository, ThreadPool threadPool, String clusterName) { this.blobStoreTransferService = blobStoreTransferService; @@ -47,6 +60,7 @@ public class RemoteClusterStateAttributesManager { this.clusterName = clusterName; this.clusterBlocksBlobStore = new RemoteClusterBlocksBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.discoveryNodesBlobStore = new RemoteDiscoveryNodesBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.customsBlobStore = new RemoteClusterStateCustomsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); } /** @@ -64,6 +78,15 @@ CheckedRunnable getAsyncMetadataWriteAction( } else if (componentData instanceof ClusterBlocks) { RemoteClusterBlocks remoteObject = new RemoteClusterBlocks((ClusterBlocks) componentData, clusterState.version(), clusterState.metadata().clusterUUID(), blobStoreRepository); return () -> clusterBlocksBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); + } else if (componentData instanceof ClusterState.Custom) { + RemoteClusterStateCustoms remoteObject = new RemoteClusterStateCustoms( + (ClusterState.Custom) componentData, + component, + clusterState.version(), + clusterState.metadata().clusterUUID(), + blobStoreRepository + ); + return () -> customsBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else { throw new RemoteStateTransferException("Remote object not found for "+ componentData.getClass()); } @@ -81,6 +104,7 @@ private ActionListener getActionListener(String component, AbstractRemoteB public CheckedRunnable getAsyncMetadataReadAction( String clusterUUID, String component, + String componentName, String uploadedFilename, LatchedActionListener listener ) { @@ -91,8 +115,26 @@ public CheckedRunnable getAsyncMetadataReadAction( } else if (component.equals(RemoteClusterBlocks.CLUSTER_BLOCKS)) { RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(uploadedFilename, clusterUUID, blobStoreRepository); return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); - } else { + } else if (component.equals(CLUSTER_STATE_CUSTOM)) { + RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(uploadedFilename, componentName, clusterUUID, blobStoreRepository); + return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, actionListener); + } else { throw new RemoteStateTransferException("Remote object not found for "+ component); } } + + public Map getUpdatedCustoms(ClusterState clusterState, ClusterState previousClusterState) { + Map updatedCustoms = new HashMap<>(); + Set currentCustoms = new HashSet<>(clusterState.customs().keySet()); + for (Map.Entry entry : previousClusterState.customs().entrySet()) { + if (currentCustoms.contains(entry.getKey()) && !entry.getValue().equals(clusterState.customs().get(entry.getKey()))) { + updatedCustoms.put(entry.getKey(), clusterState.customs().get(entry.getKey())); + } + currentCustoms.remove(entry.getKey()); + } + for (String custom : currentCustoms) { + updatedCustoms.put(custom, clusterState.customs().get(custom)); + } + return updatedCustoms; + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 2ff634f22e9cd..87d4041bc87ac 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -21,6 +21,8 @@ import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; +import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; @@ -52,6 +54,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.TemplatesMetadata; @@ -216,21 +219,16 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true, true, new ArrayList<>(clusterState.getRoutingTable().indicesRouting().values()), - true); + true, + clusterState.customs(), + true + ); final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, - uploadedMetadataResults.uploadedIndexMetadata, + uploadedMetadataResults, previousClusterUUID, - uploadedMetadataResults.uploadedCoordinationMetadata, - uploadedMetadataResults.uploadedSettingsMetadata, - uploadedMetadataResults.uploadedTransientSettingsMetadata, - uploadedMetadataResults.uploadedTemplatesMetadata, - uploadedMetadataResults.uploadedCustomMetadataMap, - uploadedMetadataResults.uploadedDiscoveryNodes, - uploadedMetadataResults.uploadedClusterBlocks, new ClusterStateDiffManifest(clusterState, ClusterState.EMPTY_STATE), - uploadedMetadataResults.uploadedIndicesRoutingMetadata, false ); @@ -278,13 +276,19 @@ public ClusterMetadataManifest writeIncrementalMetadata( assert previousClusterState.metadata().coordinationMetadata().term() == clusterState.metadata().coordinationMetadata().term(); final Map customsToBeDeletedFromRemote = new HashMap<>(previousManifest.getCustomMetadataMap()); + final Map clusterStateCustomsToBeDeleted = new HashMap<>(previousManifest.getClusterStateCustomMap()); final Map customsToUpload = remoteGlobalMetadataManager.getUpdatedCustoms(clusterState, previousClusterState); + final Map clusterStateCustomsToUpload = remoteClusterStateAttributesManager.getUpdatedCustoms(clusterState, previousClusterState); final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); for (final String custom : clusterState.metadata().customs().keySet()) { // remove all the customs which are present currently customsToBeDeletedFromRemote.remove(custom); } final Map indicesToBeDeletedFromRemote = new HashMap<>(previousClusterState.metadata().indices()); + for (final String custom : clusterState.customs().keySet()) { + // remove all the custom which are present currently + clusterStateCustomsToBeDeleted.remove(custom); + } int numIndicesUpdated = 0; int numIndicesUnchanged = 0; final Map allUploadedIndexMetadata = previousManifest.getIndices() @@ -316,7 +320,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( } List indicesRoutingToUpload = new ArrayList<>(); - if(remoteRoutingTableService!=null) { + if (remoteRoutingTableService != null) { indicesRoutingToUpload = remoteRoutingTableService.getChangedIndicesRouting(previousClusterState, clusterState); } UploadedMetadataResults uploadedMetadataResults; @@ -332,14 +336,11 @@ public ClusterMetadataManifest writeIncrementalMetadata( || Metadata.isTransientSettingsMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; boolean updateTemplatesMetadata = firstUploadForSplitGlobalMetadata || Metadata.isTemplatesMetadataEqual(previousClusterState.metadata(), clusterState.metadata()) == false; - // Write Global Metadata - final boolean updateGlobalMetadata = Metadata.isGlobalStateEquals( - previousClusterState.metadata(), - clusterState.metadata() - ) == false; // ToDo: check if these needs to be updated or not final boolean updateDiscoveryNodes = clusterState.getNodes().delta(previousClusterState.getNodes()).hasChanges(); final boolean updateClusterBlocks = !clusterState.blocks().equals(previousClusterState.blocks()); + final boolean updateHashesOfConsistentSettings = firstUploadForSplitGlobalMetadata + || Metadata.isHashesOfConsistentSettingsEqual(previousClusterState.metadata(), clusterState.metadata()) == false; // Write Index Metadata final Map previousStateIndexMetadataByName = new HashMap<>(); @@ -359,7 +360,9 @@ public ClusterMetadataManifest writeIncrementalMetadata( updateDiscoveryNodes, updateClusterBlocks, indicesRoutingToUpload, - updateTransientSettingsMetadata + updateTransientSettingsMetadata, + clusterStateCustomsToUpload, + updateHashesOfConsistentSettings ); @@ -371,27 +374,49 @@ public ClusterMetadataManifest writeIncrementalMetadata( // remove the data for removed custom/indices customsToBeDeletedFromRemote.keySet().forEach(allUploadedCustomMap::remove); indicesToBeDeletedFromRemote.keySet().forEach(allUploadedIndexMetadata::remove); + clusterStateCustomsToBeDeleted.keySet().forEach(allUploadedCustomMap::remove); List allUploadedIndicesRouting = new ArrayList<>(); if(remoteRoutingTableService!=null) { allUploadedIndicesRouting = remoteRoutingTableService.getAllUploadedIndicesRouting(previousManifest, uploadedMetadataResults.uploadedIndicesRoutingMetadata, indicesToBeDeletedFromRemote.keySet()); } + if (!updateCoordinationMetadata) { + uploadedMetadataResults.uploadedCoordinationMetadata = previousManifest.getCoordinationMetadata(); + } + if (!updateSettingsMetadata) { + uploadedMetadataResults.uploadedSettingsMetadata = previousManifest.getSettingsMetadata(); + } + if (!updateTransientSettingsMetadata) { + uploadedMetadataResults.uploadedTransientSettingsMetadata = previousManifest.getTransientSettingsMetadata(); + } + if (!updateTemplatesMetadata) { + uploadedMetadataResults.uploadedTemplatesMetadata = previousManifest.getTemplatesMetadata(); + } + if (!updateDiscoveryNodes && !firstUploadForSplitGlobalMetadata) { + uploadedMetadataResults.uploadedDiscoveryNodes = previousManifest.getDiscoveryNodesMetadata(); + } + if (!updateClusterBlocks && !firstUploadForSplitGlobalMetadata) { + uploadedMetadataResults.uploadedClusterBlocks = previousManifest.getClusterBlocksMetadata(); + } + if (!updateHashesOfConsistentSettings && !firstUploadForSplitGlobalMetadata) { + uploadedMetadataResults.uploadedHashesOfConsistentSettings = previousManifest.getHashesOfConsistentSettings(); + } + if (!firstUploadForSplitGlobalMetadata && customsToUpload.isEmpty()) { + uploadedMetadataResults.uploadedCustomMetadataMap = previousManifest.getCustomMetadataMap(); + } + if (!firstUploadForSplitGlobalMetadata && clusterStateCustomsToUpload.isEmpty()) { + uploadedMetadataResults.uploadedClusterStateCustomMetadataMap = previousManifest.getClusterStateCustomMap(); + } + uploadedMetadataResults.uploadedIndexMetadata = new ArrayList<>(allUploadedIndexMetadata.values()); + uploadedMetadataResults.uploadedIndicesRoutingMetadata = allUploadedIndicesRouting; + final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( clusterState, - new ArrayList<>(allUploadedIndexMetadata.values()), + uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), - updateCoordinationMetadata ? uploadedMetadataResults.uploadedCoordinationMetadata : previousManifest.getCoordinationMetadata(), - updateSettingsMetadata ? uploadedMetadataResults.uploadedSettingsMetadata : previousManifest.getSettingsMetadata(), - updateTransientSettingsMetadata ? uploadedMetadataResults.uploadedTransientSettingsMetadata : previousManifest.getTransientSettingsMetadata(), - updateTemplatesMetadata ? uploadedMetadataResults.uploadedTemplatesMetadata : previousManifest.getTemplatesMetadata(), - firstUploadForSplitGlobalMetadata || !customsToUpload.isEmpty() - ? allUploadedCustomMap - : previousManifest.getCustomMetadataMap(), - firstUploadForSplitGlobalMetadata || updateDiscoveryNodes ? uploadedMetadataResults.uploadedDiscoveryNodes : previousManifest.getDiscoveryNodesMetadata(), - firstUploadForSplitGlobalMetadata || updateClusterBlocks ? uploadedMetadataResults.uploadedClusterBlocks : previousManifest.getClusterBlocksMetadata(), new ClusterStateDiffManifest(clusterState, previousClusterState), - allUploadedIndicesRouting, false + false ); logger.info("MANIFEST IN INC STATE {}", manifest); @@ -462,10 +487,13 @@ private UploadedMetadataResults writeMetadataInParallel( boolean uploadDiscoveryNodes, boolean uploadClusterBlock, List indicesRoutingToUpload, - boolean uploadTransientSettingMetadata) throws IOException { + boolean uploadTransientSettingMetadata, + Map clusterStateCustomToUpload, + boolean uploadHashesOfConsistentSettings + ) throws IOException { int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size() + - (uploadTransientSettingMetadata ? 1 : 0); + (uploadTransientSettingMetadata ? 1 : 0) + clusterStateCustomToUpload.size() + (uploadHashesOfConsistentSettings ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new ConcurrentHashMap<>(totalUploadTasks); Map results = new ConcurrentHashMap<>(totalUploadTasks); @@ -555,6 +583,18 @@ private UploadedMetadataResults writeMetadataInParallel( ) ); } + if (uploadHashesOfConsistentSettings) { + uploadTasks.put( + HASHES_OF_CONSISTENT_SETTINGS, + remoteGlobalMetadataManager.getAsyncMetadataWriteAction( + new DiffableStringMap(clusterState.metadata().hashesOfConsistentSettings()), + clusterState.metadata().version(), + clusterState.metadata().clusterUUID(), + listener, + null + ) + ); + } customToUpload.forEach((key, value) -> { String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key); uploadTasks.put( @@ -574,6 +614,18 @@ private UploadedMetadataResults writeMetadataInParallel( remoteIndexMetadataManager.getIndexMetadataAsyncAction(indexMetadata, clusterState.metadata().clusterUUID(), listener) ); }); + clusterStateCustomToUpload.forEach((key, value) -> { + String customComponent = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, key); + uploadTasks.put( + customComponent, + remoteClusterStateAttributesManager.getAsyncMetadataWriteAction( + clusterState, + customComponent, + value, + listener + ) + ); + }); indicesRoutingToUpload.forEach(indexRoutingTable -> { try { @@ -641,6 +693,12 @@ private UploadedMetadataResults writeMetadataInParallel( custom, new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename()) ); + } else if (name.contains(CLUSTER_STATE_CUSTOM)) { + String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1]; + response.uploadedClusterStateCustomMetadataMap.put( + custom, + new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename()) + ); } else if (COORDINATION_METADATA.equals(name)) { response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (SETTING_METADATA.equals(name)) { @@ -655,6 +713,8 @@ private UploadedMetadataResults writeMetadataInParallel( response.uploadedDiscoveryNodes = (UploadedMetadataAttribute) uploadedMetadata; } else if (CLUSTER_BLOCKS.equals(uploadedMetadata.getComponent())) { response.uploadedClusterBlocks = (UploadedMetadataAttribute) uploadedMetadata; + } else if (HASHES_OF_CONSISTENT_SETTINGS.equals(uploadedMetadata.getComponent())) { + response.uploadedHashesOfConsistentSettings = (UploadedMetadataAttribute) uploadedMetadata; } else { throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); } @@ -733,19 +793,26 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat return null; } assert previousManifest != null : "Last cluster metadata manifest is not set"; - ClusterMetadataManifest committedManifest = remoteManifestManager.uploadManifest( - clusterState, + UploadedMetadataResults uploadedMetadataResults = new UploadedMetadataResults( previousManifest.getIndices(), - previousManifest.getPreviousClusterUUID(), + previousManifest.getCustomMetadataMap(), previousManifest.getCoordinationMetadata(), previousManifest.getSettingsMetadata(), previousManifest.getTransientSettingsMetadata(), previousManifest.getTemplatesMetadata(), - previousManifest.getCustomMetadataMap(), - previousManifest.getDiscoveryNodesMetadata(), + previousManifest.getDiscoverNodeMetadata(), previousManifest.getClusterBlocksMetadata(), + previousManifest.getIndicesRouting(), + previousManifest.getHashesOfConsistentSettings(), + previousManifest.getClusterStateCustomMap() + ); + + ClusterMetadataManifest committedManifest = remoteManifestManager.uploadManifest( + clusterState, + uploadedMetadataResults, + previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), - previousManifest.getIndicesRouting(), true + true ); if (!previousManifest.isClusterUUIDCommitted() && committedManifest.isClusterUUIDCommitted()) { remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifest); @@ -979,6 +1046,7 @@ private ClusterState readClusterStateInParallel( remoteClusterStateAttributesManager.getAsyncMetadataReadAction( clusterUUID, DISCOVERY_NODES, + DISCOVERY_NODES, manifest.getDiscoveryNodesMetadata().getUploadedFilename(), listener ) @@ -990,6 +1058,7 @@ private ClusterState readClusterStateInParallel( remoteClusterStateAttributesManager.getAsyncMetadataReadAction( clusterUUID, CLUSTER_BLOCKS, + CLUSTER_BLOCKS, manifest.getClusterBlocksMetadata().getUploadedFilename(), listener ) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index e34221c3aa53e..94a57e6b0be77 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -27,6 +27,8 @@ public class RemoteClusterStateUtils { public static final String METADATA_NAME_PLAIN_FORMAT = "%s"; public static final String METADATA_FILE_PREFIX = "metadata"; public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; + public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; + public static final String CLUSTER_STATE_EPHEMERAL_PATH_TOKEN = "ephemeral"; public static final String DELIMITER = "__"; public static final String PATH_DELIMITER = "/"; @@ -75,6 +77,7 @@ public RemoteStateTransferException(String errorDesc, Throwable cause) { public static class UploadedMetadataResults { List uploadedIndexMetadata; Map uploadedCustomMetadataMap; + Map uploadedClusterStateCustomMetadataMap; ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata; ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata; @@ -82,6 +85,7 @@ public static class UploadedMetadataResults { ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes; ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks; List uploadedIndicesRoutingMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedHashesOfConsistentSettings; public UploadedMetadataResults( List uploadedIndexMetadata, @@ -92,7 +96,9 @@ public UploadedMetadataResults( ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes, ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks, - List uploadedIndicesRoutingMetadata + List uploadedIndicesRoutingMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedHashesOfConsistentSettings, + Map uploadedClusterStateCustomMap ) { this.uploadedIndexMetadata = uploadedIndexMetadata; this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; @@ -103,6 +109,8 @@ public UploadedMetadataResults( this.uploadedDiscoveryNodes = uploadedDiscoveryNodes; this.uploadedClusterBlocks = uploadedClusterBlocks; this.uploadedIndicesRoutingMetadata = uploadedIndicesRoutingMetadata; + this.uploadedHashesOfConsistentSettings = uploadedHashesOfConsistentSettings; + this.uploadedClusterStateCustomMetadataMap = uploadedClusterStateCustomMap; } public UploadedMetadataResults() { @@ -115,6 +123,8 @@ public UploadedMetadataResults() { this.uploadedDiscoveryNodes = null; this.uploadedClusterBlocks = null; this.uploadedIndicesRoutingMetadata = new ArrayList<>(); + this.uploadedHashesOfConsistentSettings = null; + this.uploadedClusterStateCustomMetadataMap = new HashMap<>(); } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index ca186fa5711a6..8e019cdbc37f4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -9,9 +9,11 @@ package org.opensearch.gateway.remote; import static java.util.Objects.requireNonNull; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; +import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; import java.io.IOException; @@ -23,6 +25,7 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.metadata.TemplatesMetadata; @@ -39,6 +42,8 @@ import org.opensearch.gateway.remote.model.RemoteCoordinationMetadataBlobStore; import org.opensearch.gateway.remote.model.RemoteCustomMetadata; import org.opensearch.gateway.remote.model.RemoteCustomMetadataBlobStore; +import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings; +import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettingsBlobStore; import org.opensearch.gateway.remote.model.RemotePersistentSettingsBlobStore; import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata; import org.opensearch.gateway.remote.model.RemoteReadResult; @@ -53,7 +58,6 @@ public class RemoteGlobalMetadataManager { - public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; public static final String COORDINATION_METADATA = "coordination"; public static final String SETTING_METADATA = "settings"; public static final String TEMPLATES_METADATA = "templates"; @@ -106,6 +110,7 @@ public class RemoteGlobalMetadataManager { private final RemotePersistentSettingsBlobStore persistentSettingsBlobStore; private final RemoteTemplatesMetadataBlobStore templatesMetadataBlobStore; private final RemoteCustomMetadataBlobStore customMetadataBlobStore; + private final RemoteHashesOfConsistentSettingsBlobStore hashesOfConsistentSettingsBlobStore; RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, BlobStoreTransferService blobStoreTransferService, String clusterName) { @@ -119,6 +124,7 @@ public class RemoteGlobalMetadataManager { this.persistentSettingsBlobStore = new RemotePersistentSettingsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.templatesMetadataBlobStore = new RemoteTemplatesMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); this.customMetadataBlobStore = new RemoteCustomMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.hashesOfConsistentSettingsBlobStore = new RemoteHashesOfConsistentSettingsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); } @@ -149,6 +155,14 @@ CheckedRunnable getAsyncMetadataWriteAction( RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata((TemplatesMetadata) objectToUpload, metadataVersion, clusterUUID, blobStoreRepository); return () -> templatesMetadataBlobStore.writeAsync(remoteTemplatesMetadata, getActionListener(remoteTemplatesMetadata, latchedActionListener)); + } else if (objectToUpload instanceof DiffableStringMap) { + RemoteHashesOfConsistentSettings remoteObject = new RemoteHashesOfConsistentSettings( + (DiffableStringMap) objectToUpload, + metadataVersion, + clusterUUID, + blobStoreRepository + ); + return () -> hashesOfConsistentSettingsBlobStore.writeAsync(remoteObject, getActionListener(remoteObject, latchedActionListener)); } else if (objectToUpload instanceof Custom) { RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata((Custom) objectToUpload, customType ,metadataVersion, clusterUUID, blobStoreRepository); @@ -189,6 +203,9 @@ CheckedRunnable getAsyncMetadataReadAction( } else if (component.equals(CUSTOM_METADATA)) { RemoteCustomMetadata remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, blobStoreRepository); return () -> customMetadataBlobStore.readAsync(remoteBlobStoreObject, actionListener); + } else if (component.equals(HASHES_OF_CONSISTENT_SETTINGS)) { + RemoteHashesOfConsistentSettings remoteHashesOfConsistentSettings = new RemoteHashesOfConsistentSettings(uploadFilename, clusterUUID, blobStoreRepository); + return () -> hashesOfConsistentSettingsBlobStore.readAsync(remoteHashesOfConsistentSettings, actionListener); } else { throw new RemoteStateTransferException("Unknown component " + componentName); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 0bd89c73003ed..cf8ab3918e8ef 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -73,17 +73,10 @@ public class RemoteManifestManager { ClusterMetadataManifest uploadManifest( ClusterState clusterState, - List uploadedIndexMetadata, + RemoteClusterStateUtils.UploadedMetadataResults uploadedMetadataResult, String previousClusterUUID, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, - Map uploadedCustomMetadataMap, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodesMetadata, - ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocksMetadata, ClusterStateDiffManifest clusterDiffManifest, - List routingIndexMetadata, boolean committed + boolean committed ) { synchronized (this) { ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder(); @@ -95,20 +88,20 @@ ClusterMetadataManifest uploadManifest( .nodeId(nodeId) .committed(committed) .codecVersion(RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION) - .indices(uploadedIndexMetadata) + .indices(uploadedMetadataResult.uploadedIndexMetadata) .previousClusterUUID(previousClusterUUID) .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) - .coordinationMetadata(uploadedCoordinationMetadata) - .settingMetadata(uploadedSettingsMetadata) - .templatesMetadata(uploadedTemplatesMetadata) - .customMetadataMap(uploadedCustomMetadataMap) - .discoveryNodesMetadata(uploadedDiscoveryNodesMetadata) - .clusterBlocksMetadata(uploadedClusterBlocksMetadata) + .coordinationMetadata(uploadedMetadataResult.uploadedCoordinationMetadata) + .settingMetadata(uploadedMetadataResult.uploadedSettingsMetadata) + .templatesMetadata(uploadedMetadataResult.uploadedTemplatesMetadata) + .customMetadataMap(uploadedMetadataResult.uploadedCustomMetadataMap) + .discoveryNodesMetadata(uploadedMetadataResult.uploadedDiscoveryNodes) + .clusterBlocksMetadata(uploadedMetadataResult.uploadedClusterBlocks) .diffManifest(clusterDiffManifest) .routingTableVersion(clusterState.getRoutingTable().version()) - .indicesRouting(routingIndexMetadata) + .indicesRouting(uploadedMetadataResult.uploadedIndicesRoutingMetadata) .metadataVersion(clusterState.metadata().version()) - .transientSettingsMetadata(uploadedTransientSettingsMetadata); + .transientSettingsMetadata(uploadedMetadataResult.uploadedTransientSettingsMetadata); final ClusterMetadataManifest manifest = manifestBuilder.build(); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest); return manifest; diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java new file mode 100644 index 0000000000000..c248b2823246f --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.common.io.Streams; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; + +public class RemoteClusterStateCustoms extends AbstractRemoteBlobObject { + public static final String CLUSTER_STATE_CUSTOM = "cluster-state-custom"; + + public final ChecksumBlobStoreFormat clusterStateCustomBlobStoreFormat; + private long stateVersion; + private String customType; + private ClusterState.Custom custom; + + public RemoteClusterStateCustoms(ClusterState.Custom custom, String customType, long stateVersion, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); + this.stateVersion = stateVersion; + this.customType = customType; + this.custom = custom; + this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( + CLUSTER_STATE_CUSTOM, + METADATA_NAME_FORMAT, + parser -> ClusterState.Custom.fromXContent(parser, customType) + ); + } + + public RemoteClusterStateCustoms(String blobName, String customType, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); + this.blobName = blobName; + this.customType = customType; + this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( + CLUSTER_STATE_CUSTOM, + METADATA_NAME_FORMAT, + parser -> ClusterState.Custom.fromXContent(parser, customType) + ); + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), CLUSTER_STATE_CUSTOM); + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new ClusterMetadataManifest.UploadedMetadataAttribute(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType), blobName); + } + + @Override + public ClusterState.Custom get() { + return custom; + } + + @Override + public InputStream serialize() throws IOException { + return clusterStateCustomBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public ClusterState.Custom deserialize(InputStream inputStream) throws IOException { + return clusterStateCustomBlobStoreFormat.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java new file mode 100644 index 0000000000000..2718347a05f47 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteClusterStateCustomsBlobStore extends AbstractRemoteBlobStore { + public RemoteClusterStateCustomsBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java index 81e7794fc6949..b5596af7ab1e9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java @@ -9,6 +9,7 @@ package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; @@ -72,7 +73,7 @@ public RemoteCustomMetadata(String blobName, String customType, String clusterUU @Override public BlobPathParameters getBlobPathParameters() { String prefix = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, customType); - return new BlobPathParameters(List.of("global-metadata"), prefix); + return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), prefix); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index 568fce6ac0007..07b272b5edfaa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -9,6 +9,7 @@ package org.opensearch.gateway.remote.model; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; @@ -52,12 +53,12 @@ public RemoteDiscoveryNodes(String blobName, String clusterUUID, BlobStoreReposi @Override public BlobPathParameters getBlobPathParameters() { - return new BlobPathParameters(List.of("transient"), DISCOVERY_NODES); + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), DISCOVERY_NODES); } @Override public String generateBlobFileName() { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/transient/______ + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ String blobFileName = String.join( DELIMITER, getBlobPathParameters().getFilePrefix(), diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java new file mode 100644 index 0000000000000..1cd9e48e07927 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.io.Streams; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +public class RemoteHashesOfConsistentSettings extends AbstractRemoteBlobObject { + public static final String HASHES_OF_CONSISTENT_SETTINGS = "hashes-of-consistent-settings"; + public static final ChecksumBlobStoreFormat HASHES_OF_CONSISTENT_SETTINGS_FORMAT = new ChecksumBlobStoreFormat<>( + HASHES_OF_CONSISTENT_SETTINGS, + METADATA_NAME_FORMAT, + DiffableStringMap::fromXContent + ); + + private DiffableStringMap hashesOfConsistentSettings; + private long metadataVersion; + public RemoteHashesOfConsistentSettings(DiffableStringMap hashesOfConsistentSettings, long metadataVersion, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); + this.metadataVersion = metadataVersion; + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + } + + public RemoteHashesOfConsistentSettings(String blobName, String clusterUUID, BlobStoreRepository blobStoreRepository) { + super(blobStoreRepository, clusterUUID); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), HASHES_OF_CONSISTENT_SETTINGS); + } + + @Override + public String generateBlobFileName() { + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new ClusterMetadataManifest.UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, blobName); + } + + @Override + public DiffableStringMap get() { + return hashesOfConsistentSettings; + } + + @Override + public InputStream serialize() throws IOException { + return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.serialize(hashesOfConsistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public DiffableStringMap deserialize(InputStream inputStream) throws IOException { + return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsBlobStore.java new file mode 100644 index 0000000000000..a05a21f6128dd --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsBlobStore.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteHashesOfConsistentSettingsBlobStore extends AbstractRemoteBlobStore { + public RemoteHashesOfConsistentSettingsBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { + super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index a9b4b67d0e215..9ce4e90238bf0 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -60,10 +60,10 @@ import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.nodesWithLocalNodeClusterManager; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.encodeString; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; From bb335f6da175657a9ef082e3a5f44c9ee3c0d13b Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sun, 2 Jun 2024 22:47:38 +0530 Subject: [PATCH 108/133] Address remote state interface comments --- .../remote/RemoteClusterStateServiceIT.java | 8 +- .../AbstractRemoteWritableBlobEntity.java} | 28 +-- .../remote}/BlobPathParameters.java | 4 +- .../remote/RemoteWritableEntityStore.java} | 11 +- .../remote/RemoteWriteableEntity.java} | 17 +- .../common/remote/package-info.java | 11 + .../RemoteClusterStateAttributesManager.java | 60 +++--- .../remote/RemoteClusterStateService.java | 136 ++++++++---- .../remote/RemoteGlobalMetadataManager.java | 202 ++++++++---------- .../remote/RemoteIndexMetadataManager.java | 35 ++- .../gateway/remote/RemoteManifestManager.java | 62 +++--- .../remote/model/RemoteClusterBlocks.java | 26 ++- .../model/RemoteClusterBlocksBlobStore.java | 25 --- .../model/RemoteClusterMetadataManifest.java | 28 ++- ...emoteClusterMetadataManifestBlobStore.java | 25 --- ....java => RemoteClusterStateBlobStore.java} | 46 ++-- .../model/RemoteClusterStateCustoms.java | 25 ++- .../RemoteClusterStateCustomsBlobStore.java | 20 -- .../model/RemoteCoordinationMetadata.java | 24 ++- .../RemoteCoordinationMetadataBlobStore.java | 25 --- .../remote/model/RemoteCustomMetadata.java | 28 ++- .../model/RemoteCustomMetadataBlobStore.java | 25 --- .../remote/model/RemoteDiscoveryNodes.java | 24 ++- .../model/RemoteDiscoveryNodesBlobStore.java | 25 --- .../remote/model/RemoteGlobalMetadata.java | 75 +++++++ .../RemoteHashesOfConsistentSettings.java | 42 ++-- ...teHashesOfConsistentSettingsBlobStore.java | 20 -- .../remote/model/RemoteIndexMetadata.java | 26 ++- .../model/RemoteIndexMetadataBlobStore.java | 25 --- .../RemotePersistentSettingsBlobStore.java | 25 --- .../RemotePersistentSettingsMetadata.java | 24 ++- .../remote/model/RemoteTemplatesMetadata.java | 24 ++- .../RemoteTemplatesMetadataBlobStore.java | 25 --- .../RemoteTransientSettingsBlobStore.java | 25 --- .../RemoteTransientSettingsMetadata.java | 24 ++- .../remote/ClusterMetadataManifestTests.java | 28 +-- ...RemoteClusterStateCleanupManagerTests.java | 6 +- .../RemoteClusterStateServiceTests.java | 28 +-- .../RemoteGlobalMetadataManagerTests.java | 60 +++++- .../RemoteIndexMetadataManagerTests.java | 30 ++- .../remote/RemoteManifestManagerTests.java | 21 +- .../remote/model/RemoteClusterBlocksTest.java | 44 ++-- .../RemoteClusterMetadataManifestTests.java | 44 ++-- .../model/RemoteIndexMetadataTests.java | 46 ++-- 44 files changed, 823 insertions(+), 739 deletions(-) rename server/src/main/java/org/opensearch/{gateway/remote/model/AbstractRemoteBlobObject.java => common/remote/AbstractRemoteWritableBlobEntity.java} (63%) rename server/src/main/java/org/opensearch/{gateway/remote/model => common/remote}/BlobPathParameters.java (83%) rename server/src/main/java/org/opensearch/{gateway/remote/model/RemoteObjectStore.java => common/remote/RemoteWritableEntityStore.java} (56%) rename server/src/main/java/org/opensearch/{gateway/remote/model/RemoteObject.java => common/remote/RemoteWriteableEntity.java} (63%) create mode 100644 server/src/main/java/org/opensearch/common/remote/package-info.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java rename server/src/main/java/org/opensearch/gateway/remote/model/{AbstractRemoteBlobStore.java => RemoteClusterStateBlobStore.java} (57%) delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesBlobStore.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataBlobStore.java delete mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java index 4fb09797bd4d1..8d7b71ac7bb83 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java @@ -29,10 +29,10 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_FILE_PREFIX; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; +import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA; +import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase { diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java similarity index 63% rename from server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java rename to server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java index b547d3584382a..2ef37a6af35c2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobObject.java +++ b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java @@ -6,31 +6,33 @@ * compatible open source license. */ -package org.opensearch.gateway.remote.model; +package org.opensearch.common.remote; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; -import org.opensearch.repositories.blobstore.BlobStoreRepository; /** - * An extension of {@link RemoteObject} class which caters to the use case of writing to and reading from a blob storage + * An extension of {@link RemoteWriteableEntity} class which caters to the use case of writing to and reading from a blob storage * * @param The class type which can be uploaded to or downloaded from a blob storage. */ -public abstract class AbstractRemoteBlobObject implements RemoteObject { +public abstract class AbstractRemoteWritableBlobEntity implements RemoteWriteableEntity { protected String blobFileName; protected String blobName; - private final BlobStoreRepository blobStoreRepository; private final String clusterUUID; + private final Compressor compressor; + private final NamedXContentRegistry namedXContentRegistry; - public AbstractRemoteBlobObject(BlobStoreRepository blobStoreRepository, String clusterUUID) { - this.blobStoreRepository = blobStoreRepository; + public AbstractRemoteWritableBlobEntity(final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { this.clusterUUID = clusterUUID; + this.compressor = compressor; + this.namedXContentRegistry = namedXContentRegistry; } public abstract BlobPathParameters getBlobPathParameters(); @@ -45,6 +47,9 @@ public String getBlobFileName() { return null; } String[] pathTokens = blobName.split(PATH_DELIMITER); + if (pathTokens.length < 1) { + return null; + } blobFileName = pathTokens[pathTokens.length - 1]; } return blobFileName; @@ -62,12 +67,11 @@ public void setFullBlobName(BlobPath blobPath) { this.blobName = blobPath.buildAsString() + blobFileName; } - protected Compressor getCompressor() { - return blobStoreRepository.getCompressor(); + public NamedXContentRegistry getNamedXContentRegistry() { + return namedXContentRegistry; } - - protected BlobStoreRepository getBlobStoreRepository() { - return this.blobStoreRepository; + protected Compressor getCompressor() { + return compressor; } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/BlobPathParameters.java b/server/src/main/java/org/opensearch/common/remote/BlobPathParameters.java similarity index 83% rename from server/src/main/java/org/opensearch/gateway/remote/model/BlobPathParameters.java rename to server/src/main/java/org/opensearch/common/remote/BlobPathParameters.java index cc7a2c18b8792..58c73a804b66a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/BlobPathParameters.java +++ b/server/src/main/java/org/opensearch/common/remote/BlobPathParameters.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote.model; +package org.opensearch.common.remote; import java.util.List; @@ -19,7 +19,7 @@ public class BlobPathParameters { private final List pathTokens; private final String filePrefix; - public BlobPathParameters(List pathTokens, String filePrefix) { + public BlobPathParameters(final List pathTokens, final String filePrefix) { this.pathTokens = pathTokens; this.filePrefix = filePrefix; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java b/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java similarity index 56% rename from server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java rename to server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java index 92e494343b664..48db26292f6ae 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObjectStore.java +++ b/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.gateway.remote.model; +package org.opensearch.common.remote; import java.io.IOException; import org.opensearch.core.action.ActionListener; @@ -15,12 +15,13 @@ * An interface to read/write an object from/to a remote storage. This interface is agnostic of the remote storage type. * * @param The object type which can be uploaded to or downloaded from remote storage. + * @param The wrapper entity which provides methods for serializing/deserializing entity T. */ -public interface RemoteObjectStore> { +public interface RemoteWritableEntityStore> { - public void writeAsync(U obj, ActionListener listener); + public void writeAsync(U entity, ActionListener listener); - public T read(U obj) throws IOException; + public U read(U entity) throws IOException; - public void readAsync(U obj, ActionListener listener); + public void readAsync(U entity, ActionListener listener); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObject.java b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntity.java similarity index 63% rename from server/src/main/java/org/opensearch/gateway/remote/model/RemoteObject.java rename to server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntity.java index 9fb4738ff0efa..06818489a1f32 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteObject.java +++ b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntity.java @@ -6,20 +6,23 @@ * compatible open source license. */ -package org.opensearch.gateway.remote.model; +package org.opensearch.common.remote; import java.io.IOException; import java.io.InputStream; -import org.opensearch.common.CheckedRunnable; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; /** - * An interface to read/write and object from/to a remote storage. This interface is agnostic of the remote storage type. + * An interface to which provides defines the serialization/deserialization methods for objects to be uploaded to or downloaded from remote store. + * This interface is agnostic of the remote storage type. * - * @param The object type which can be upload to or download from remote storage. + * @param The object type which can be uploaded to or downloaded from remote storage. */ -public interface RemoteObject { +public interface RemoteWriteableEntity { + + /** + * @param object The object T which is to be set in this writable entity + */ + public void set(T object); /** * @return The entity T contained within this class diff --git a/server/src/main/java/org/opensearch/common/remote/package-info.java b/server/src/main/java/org/opensearch/common/remote/package-info.java new file mode 100644 index 0000000000000..08ff9e910dc98 --- /dev/null +++ b/server/src/main/java/org/opensearch/common/remote/package-info.java @@ -0,0 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/** + * Common remote store package + */ +package org.opensearch.common.remote; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 99e48d44b83ca..c02b5b3116f7a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -8,59 +8,50 @@ package org.opensearch.gateway.remote; +import java.io.IOException; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.cluster.block.ClusterBlocks; -import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.CheckedRunnable; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import org.opensearch.gateway.remote.model.AbstractRemoteBlobObject; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.gateway.remote.model.RemoteClusterBlocks; -import org.opensearch.gateway.remote.model.RemoteClusterBlocksBlobStore; import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms; -import org.opensearch.gateway.remote.model.RemoteClusterStateCustomsBlobStore; import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes; -import org.opensearch.gateway.remote.model.RemoteDiscoveryNodesBlobStore; -import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings; -import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettingsBlobStore; import org.opensearch.gateway.remote.model.RemoteReadResult; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; -import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; -import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; public class RemoteClusterStateAttributesManager { public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; public static final String DISCOVERY_NODES = "nodes"; public static final String CLUSTER_BLOCKS = "blocks"; public static final int CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION = 1; - private final BlobStoreTransferService blobStoreTransferService; - private final BlobStoreRepository blobStoreRepository; - private final ThreadPool threadPool; - private final String clusterName; - private final RemoteClusterBlocksBlobStore clusterBlocksBlobStore; - private final RemoteDiscoveryNodesBlobStore discoveryNodesBlobStore; - private final RemoteClusterStateCustomsBlobStore customsBlobStore; + private final RemoteClusterStateBlobStore clusterBlocksBlobStore; + private final RemoteClusterStateBlobStore discoveryNodesBlobStore; + private final RemoteClusterStateBlobStore customsBlobStore; + private final Compressor compressor; + private final NamedXContentRegistry namedXContentRegistry; - RemoteClusterStateAttributesManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository repository, ThreadPool threadPool, String clusterName) { - this.blobStoreTransferService = blobStoreTransferService; - this.blobStoreRepository = repository; - this.threadPool = threadPool; - this.clusterName = clusterName; - this.clusterBlocksBlobStore = new RemoteClusterBlocksBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.discoveryNodesBlobStore = new RemoteDiscoveryNodesBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.customsBlobStore = new RemoteClusterStateCustomsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + RemoteClusterStateAttributesManager( + RemoteClusterStateBlobStore clusterBlocksBlobStore, RemoteClusterStateBlobStore discoveryNodesBlobStore, RemoteClusterStateBlobStore customsBlobStore, Compressor compressor, NamedXContentRegistry namedXContentRegistry) { + this.clusterBlocksBlobStore = clusterBlocksBlobStore; + this.discoveryNodesBlobStore = discoveryNodesBlobStore; + this.customsBlobStore = customsBlobStore; + this.compressor = compressor; + this.namedXContentRegistry = namedXContentRegistry; } /** @@ -73,10 +64,10 @@ CheckedRunnable getAsyncMetadataWriteAction( LatchedActionListener latchedActionListener ) { if (componentData instanceof DiscoveryNodes) { - RemoteDiscoveryNodes remoteObject = new RemoteDiscoveryNodes((DiscoveryNodes)componentData, clusterState.version(), clusterState.metadata().clusterUUID(), blobStoreRepository); + RemoteDiscoveryNodes remoteObject = new RemoteDiscoveryNodes((DiscoveryNodes)componentData, clusterState.version(), clusterState.metadata().clusterUUID(), compressor, namedXContentRegistry); return () -> discoveryNodesBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else if (componentData instanceof ClusterBlocks) { - RemoteClusterBlocks remoteObject = new RemoteClusterBlocks((ClusterBlocks) componentData, clusterState.version(), clusterState.metadata().clusterUUID(), blobStoreRepository); + RemoteClusterBlocks remoteObject = new RemoteClusterBlocks((ClusterBlocks) componentData, clusterState.version(), clusterState.metadata().clusterUUID(), compressor, namedXContentRegistry); return () -> clusterBlocksBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else if (componentData instanceof ClusterState.Custom) { RemoteClusterStateCustoms remoteObject = new RemoteClusterStateCustoms( @@ -84,7 +75,8 @@ CheckedRunnable getAsyncMetadataWriteAction( component, clusterState.version(), clusterState.metadata().clusterUUID(), - blobStoreRepository + compressor, + namedXContentRegistry ); return () -> customsBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else { @@ -92,7 +84,7 @@ CheckedRunnable getAsyncMetadataWriteAction( } } - private ActionListener getActionListener(String component, AbstractRemoteBlobObject remoteObject, LatchedActionListener latchedActionListener) { + private ActionListener getActionListener(String component, AbstractRemoteWritableBlobEntity remoteObject, LatchedActionListener latchedActionListener) { return ActionListener.wrap( resp -> latchedActionListener.onResponse( remoteObject.getUploadedMetadata() @@ -110,13 +102,13 @@ public CheckedRunnable getAsyncMetadataReadAction( ) { ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); if (component.equals(RemoteDiscoveryNodes.DISCOVERY_NODES)) { - RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(uploadedFilename, clusterUUID, blobStoreRepository); + RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); return () -> discoveryNodesBlobStore.readAsync(remoteDiscoveryNodes, actionListener); } else if (component.equals(RemoteClusterBlocks.CLUSTER_BLOCKS)) { - RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(uploadedFilename, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); } else if (component.equals(CLUSTER_STATE_CUSTOM)) { - RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(uploadedFilename, componentName, clusterUUID, blobStoreRepository); + RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(uploadedFilename, componentName, clusterUUID, compressor, namedXContentRegistry); return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, actionListener); } else { throw new RemoteStateTransferException("Remote object not found for "+ component); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 87d4041bc87ac..32e8761691129 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -13,14 +13,13 @@ import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; - import static org.opensearch.gateway.remote.RemoteClusterStateUtils.UploadedMetadataResults; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.clusterUUIDContainer; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.COORDINATION_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_DELIMITER; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.CUSTOM_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.SETTING_METADATA; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; +import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA; +import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; @@ -57,12 +56,13 @@ import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedRunnable; import org.opensearch.common.Nullable; @@ -79,7 +79,21 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.model.RemoteClusterBlocks; +import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; +import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms; +import org.opensearch.gateway.remote.model.RemoteCoordinationMetadata; +import org.opensearch.gateway.remote.model.RemoteCustomMetadata; +import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes; +import org.opensearch.gateway.remote.model.RemoteGlobalMetadata; +import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings; +import org.opensearch.gateway.remote.model.RemoteIndexMetadata; +import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata; import org.opensearch.gateway.remote.model.RemoteReadResult; +import org.opensearch.gateway.remote.model.RemoteTemplatesMetadata; +import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; +import org.opensearch.common.remote.RemoteWritableEntityStore; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -94,6 +108,7 @@ * @opensearch.internal */ public class RemoteClusterStateService implements Closeable { + public static final int RETAINED_MANIFESTS = 10; private static final Logger logger = LogManager.getLogger(RemoteClusterStateService.class); @@ -151,6 +166,7 @@ public class RemoteClusterStateService implements Closeable { params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY); FORMAT_PARAMS = new ToXContent.MapParams(params); } + private String latestClusterName; private String latestClusterUUID; private long lastCleanupAttemptState; @@ -180,7 +196,7 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteClusterStateEnabled); this.remoteStateStats = new RemotePersistenceStats(); - if(isRemoteRoutingTableEnabled(settings)) { + if (isRemoteRoutingTableEnabled(settings)) { this.remoteRoutingTableService = new RemoteRoutingTableService(repositoriesService, settings, threadPool); logger.info("REMOTE ROUTING ENABLED"); @@ -278,7 +294,8 @@ public ClusterMetadataManifest writeIncrementalMetadata( final Map customsToBeDeletedFromRemote = new HashMap<>(previousManifest.getCustomMetadataMap()); final Map clusterStateCustomsToBeDeleted = new HashMap<>(previousManifest.getClusterStateCustomMap()); final Map customsToUpload = remoteGlobalMetadataManager.getUpdatedCustoms(clusterState, previousClusterState); - final Map clusterStateCustomsToUpload = remoteClusterStateAttributesManager.getUpdatedCustoms(clusterState, previousClusterState); + final Map clusterStateCustomsToUpload = remoteClusterStateAttributesManager.getUpdatedCustoms(clusterState, + previousClusterState); final Map allUploadedCustomMap = new HashMap<>(previousManifest.getCustomMetadataMap()); for (final String custom : clusterState.metadata().customs().keySet()) { // remove all the customs which are present currently @@ -348,7 +365,6 @@ public ClusterMetadataManifest writeIncrementalMetadata( previousStateIndexMetadataByName.put(indexMetadata.getIndex().getName(), indexMetadata); } - uploadedMetadataResults = writeMetadataInParallel( clusterState, toUpload, @@ -365,7 +381,6 @@ public ClusterMetadataManifest writeIncrementalMetadata( updateHashesOfConsistentSettings ); - // update the map if the metadata was uploaded uploadedMetadataResults.uploadedIndexMetadata.forEach( uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata) @@ -377,8 +392,9 @@ public ClusterMetadataManifest writeIncrementalMetadata( clusterStateCustomsToBeDeleted.keySet().forEach(allUploadedCustomMap::remove); List allUploadedIndicesRouting = new ArrayList<>(); - if(remoteRoutingTableService!=null) { - allUploadedIndicesRouting = remoteRoutingTableService.getAllUploadedIndicesRouting(previousManifest, uploadedMetadataResults.uploadedIndicesRoutingMetadata, indicesToBeDeletedFromRemote.keySet()); + if (remoteRoutingTableService != null) { + allUploadedIndicesRouting = remoteRoutingTableService.getAllUploadedIndicesRouting(previousManifest, + uploadedMetadataResults.uploadedIndicesRoutingMetadata, indicesToBeDeletedFromRemote.keySet()); } if (!updateCoordinationMetadata) { @@ -491,9 +507,10 @@ private UploadedMetadataResults writeMetadataInParallel( Map clusterStateCustomToUpload, boolean uploadHashesOfConsistentSettings ) throws IOException { - int totalUploadTasks = indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata - ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size() + - (uploadTransientSettingMetadata ? 1 : 0) + clusterStateCustomToUpload.size() + (uploadHashesOfConsistentSettings ? 1 : 0); + int totalUploadTasks = + indexToUpload.size() + indexMetadataUploadListeners.size() + customToUpload.size() + (uploadCoordinationMetadata ? 1 : 0) + (uploadSettingsMetadata + ? 1 : 0) + (uploadTemplateMetadata ? 1 : 0) + (uploadDiscoveryNodes ? 1 : 0) + (uploadClusterBlock ? 1 : 0) + indicesRoutingToUpload.size() + + (uploadTransientSettingMetadata ? 1 : 0) + clusterStateCustomToUpload.size() + (uploadHashesOfConsistentSettings ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalUploadTasks); Map> uploadTasks = new ConcurrentHashMap<>(totalUploadTasks); Map results = new ConcurrentHashMap<>(totalUploadTasks); @@ -699,21 +716,21 @@ private UploadedMetadataResults writeMetadataInParallel( custom, new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename()) ); - } else if (COORDINATION_METADATA.equals(name)) { + } else if (RemoteCoordinationMetadata.COORDINATION_METADATA.equals(name)) { response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata; - } else if (SETTING_METADATA.equals(name)) { + } else if (RemotePersistentSettingsMetadata.SETTING_METADATA.equals(name)) { response.uploadedSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; - } else if (TRANSIENT_SETTING_METADATA.equals(name)) { + } else if (RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA.equals(name)) { response.uploadedTransientSettingsMetadata = (UploadedMetadataAttribute) uploadedMetadata; - } else if (TEMPLATES_METADATA.equals(name)) { + } else if (RemoteTemplatesMetadata.TEMPLATES_METADATA.equals(name)) { response.uploadedTemplatesMetadata = (UploadedMetadataAttribute) uploadedMetadata; } else if (name.contains(UploadedIndexMetadata.COMPONENT_PREFIX)) { response.uploadedIndexMetadata.add((UploadedIndexMetadata) uploadedMetadata); - } else if (DISCOVERY_NODES.equals(uploadedMetadata.getComponent())) { + } else if (RemoteDiscoveryNodes.DISCOVERY_NODES.equals(uploadedMetadata.getComponent())) { response.uploadedDiscoveryNodes = (UploadedMetadataAttribute) uploadedMetadata; - } else if (CLUSTER_BLOCKS.equals(uploadedMetadata.getComponent())) { + } else if (RemoteClusterBlocks.CLUSTER_BLOCKS.equals(uploadedMetadata.getComponent())) { response.uploadedClusterBlocks = (UploadedMetadataAttribute) uploadedMetadata; - } else if (HASHES_OF_CONSISTENT_SETTINGS.equals(uploadedMetadata.getComponent())) { + } else if (RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS.equals(uploadedMetadata.getComponent())) { response.uploadedHashesOfConsistentSettings = (UploadedMetadataAttribute) uploadedMetadata; } else { throw new IllegalStateException("Unexpected metadata component " + uploadedMetadata.getComponent()); @@ -842,7 +859,7 @@ public void close() throws IOException { if (blobStoreRepository != null) { IOUtils.close(blobStoreRepository); } - if(this.remoteRoutingTableService != null) { + if (this.remoteRoutingTableService != null) { this.remoteRoutingTableService.close(); } } @@ -856,14 +873,45 @@ public void start() { final Repository repository = repositoriesService.get().repository(remoteStoreRepo); assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository"; blobStoreRepository = (BlobStoreRepository) repository; - if(this.remoteRoutingTableService != null) { + if (this.remoteRoutingTableService != null) { this.remoteRoutingTableService.start(); } String clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings).value(); - remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings,threadpool, getBlobStoreTransferService(), clusterName); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings,threadpool, clusterName, getBlobStoreTransferService()); - remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(getBlobStoreTransferService(), blobStoreRepository, threadpool, clusterName); - remoteManifestManager = new RemoteManifestManager(getBlobStoreTransferService(), blobStoreRepository, clusterSettings, nodeId, threadpool, clusterName); + RemoteWritableEntityStore globalMetadataBlobStore = new RemoteClusterStateBlobStore<>(getBlobStoreTransferService(), + blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore coordinationMetadataBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore transientSettingsBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore persistentSettingsBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore templatesMetadataBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore customMetadataBlobStore = new RemoteClusterStateBlobStore<>(getBlobStoreTransferService(), + blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore hashesOfConsistentSettingsBlobStore = + new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(clusterSettings, globalMetadataBlobStore, coordinationMetadataBlobStore, + transientSettingsBlobStore, persistentSettingsBlobStore, templatesMetadataBlobStore, customMetadataBlobStore, hashesOfConsistentSettingsBlobStore, + blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry()); + RemoteClusterStateBlobStore indexMetadataBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(indexMetadataBlobStore, clusterSettings, blobStoreRepository.getCompressor(), + blobStoreRepository.getNamedXContentRegistry()); + RemoteClusterStateBlobStore clusterBlocksBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore discoveryNodesBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore clusterStateCustomsBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(clusterBlocksBlobStore, discoveryNodesBlobStore, + clusterStateCustomsBlobStore, + blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry()); + RemoteClusterStateBlobStore manifestBlobStore = new RemoteClusterStateBlobStore<>( + getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); + remoteManifestManager = new RemoteManifestManager(manifestBlobStore, clusterSettings, nodeId, blobStoreRepository.getCompressor(), + blobStoreRepository.getNamedXContentRegistry(), blobStoreRepository); remoteClusterStateCleanupManager.start(); } @@ -926,7 +974,9 @@ private ClusterState readClusterStateInParallel( boolean readClusterBlocks, List indicesRoutingToRead ) throws IOException { - int totalReadTasks = indicesToRead.size() + customToRead.size() + indicesRoutingToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + (readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0) + (readTransientSettingsMetadata ? 1 : 0); + int totalReadTasks = + indicesToRead.size() + customToRead.size() + indicesRoutingToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + ( + readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0) + (readTransientSettingsMetadata ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalReadTasks); List> asyncMetadataReadActions = new ArrayList<>(); List readResults = new ArrayList<>(); @@ -971,7 +1021,7 @@ private ClusterState readClusterStateInParallel( latch ); - for (UploadedIndexMetadata indexRouting: indicesRoutingToRead) { + for (UploadedIndexMetadata indexRouting : indicesRoutingToRead) { asyncMetadataReadActions.add( remoteRoutingTableService.getAsyncIndexMetadataReadAction( indexRouting.getUploadedFilename(), @@ -1147,7 +1197,8 @@ private ClusterState readClusterStateInParallel( .build(); } - public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId, boolean includeEphemeral) throws IOException { + public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId, boolean includeEphemeral) + throws IOException { return readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), manifest, @@ -1166,16 +1217,19 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada ); } - public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) throws IOException { + public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) + throws IOException { assert manifest.getDiffManifest() != null; ClusterStateDiffManifest diff = manifest.getDiffManifest(); List updatedIndices = diff.getIndicesUpdated().stream().map(idx -> { - Optional uploadedIndexMetadataOptional = manifest.getIndices().stream().filter(idx2 -> idx2.getIndexName().equals(idx)).findFirst(); + Optional uploadedIndexMetadataOptional = manifest.getIndices().stream().filter(idx2 -> idx2.getIndexName().equals(idx)) + .findFirst(); assert uploadedIndexMetadataOptional.isPresent() == true; return uploadedIndexMetadataOptional.get(); }).collect(Collectors.toList()); - List updatedIndexRouting = remoteRoutingTableService.getUpdatedIndexRoutingTableMetadata(diff.getIndicesRoutingUpdated(), manifest.getIndicesRouting()); + List updatedIndexRouting = remoteRoutingTableService.getUpdatedIndexRoutingTableMetadata(diff.getIndicesRoutingUpdated(), + manifest.getIndicesRouting()); Map updatedCustomMetadata = new HashMap<>(); if (diff.getCustomMetadataUpdated() != null) { @@ -1202,7 +1256,7 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); // remove the deleted indices from the metadata - for (String index:diff.getIndicesDeleted()) { + for (String index : diff.getIndicesDeleted()) { metadataBuilder.remove(index); } // remove the deleted metadata customs from the metadata @@ -1214,7 +1268,7 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata HashMap indexRoutingTables = new HashMap<>(updatedClusterState.getRoutingTable().getIndicesRouting()); - for(String indexName: diff.getIndicesRoutingDeleted()){ + for (String indexName : diff.getIndicesRoutingDeleted()) { indexRoutingTables.remove(indexName); } @@ -1337,9 +1391,9 @@ private List createClusterChain(final Map trimClusterUUIDs( diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index 8e019cdbc37f4..fb76bd3dc4f66 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -9,10 +9,7 @@ package org.opensearch.gateway.remote; import static java.util.Objects.requireNonNull; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; @@ -30,40 +27,28 @@ import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.metadata.TemplatesMetadata; import org.opensearch.common.CheckedRunnable; -import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.gateway.remote.model.AbstractRemoteBlobObject; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.gateway.remote.model.RemoteCoordinationMetadata; -import org.opensearch.gateway.remote.model.RemoteCoordinationMetadataBlobStore; import org.opensearch.gateway.remote.model.RemoteCustomMetadata; -import org.opensearch.gateway.remote.model.RemoteCustomMetadataBlobStore; +import org.opensearch.gateway.remote.model.RemoteGlobalMetadata; import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings; -import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettingsBlobStore; -import org.opensearch.gateway.remote.model.RemotePersistentSettingsBlobStore; import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata; import org.opensearch.gateway.remote.model.RemoteReadResult; import org.opensearch.gateway.remote.model.RemoteTemplatesMetadata; -import org.opensearch.gateway.remote.model.RemoteTemplatesMetadataBlobStore; -import org.opensearch.gateway.remote.model.RemoteTransientSettingsBlobStore; import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import org.opensearch.threadpool.ThreadPool; +import org.opensearch.common.remote.RemoteWritableEntityStore; public class RemoteGlobalMetadataManager { - public static final String COORDINATION_METADATA = "coordination"; - public static final String SETTING_METADATA = "settings"; - public static final String TEMPLATES_METADATA = "templates"; - public static final String CUSTOM_METADATA = "custom"; - public static final String CUSTOM_DELIMITER = "--"; - public static final TimeValue GLOBAL_METADATA_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); public static final Setting GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING = Setting.timeSetting( @@ -73,58 +58,39 @@ public class RemoteGlobalMetadataManager { Setting.Property.NodeScope ); - public static final ChecksumBlobStoreFormat GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "metadata", - METADATA_NAME_FORMAT, - Metadata::fromXContent - ); - - public static final ChecksumBlobStoreFormat COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "coordination", - METADATA_NAME_FORMAT, - CoordinationMetadata::fromXContent - ); - - public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "settings", - METADATA_NAME_FORMAT, - Settings::fromXContent - ); - - public static final ChecksumBlobStoreFormat TEMPLATES_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( - "templates", - METADATA_NAME_FORMAT, - TemplatesMetadata::fromXContent - ); - public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; - private final BlobStoreRepository blobStoreRepository; - private final ThreadPool threadPool; - private volatile TimeValue globalMetadataUploadTimeout; - private final BlobStoreTransferService blobStoreTransferService; - private final String clusterName; - private final RemoteCoordinationMetadataBlobStore coordinationMetadataBlobStore; - private final RemoteTransientSettingsBlobStore transientSettingsBlobStore; - private final RemotePersistentSettingsBlobStore persistentSettingsBlobStore; - private final RemoteTemplatesMetadataBlobStore templatesMetadataBlobStore; - private final RemoteCustomMetadataBlobStore customMetadataBlobStore; - private final RemoteHashesOfConsistentSettingsBlobStore hashesOfConsistentSettingsBlobStore; + private final RemoteWritableEntityStore globalMetadataBlobStore; + private final RemoteClusterStateBlobStore coordinationMetadataBlobStore; + private final RemoteClusterStateBlobStore transientSettingsBlobStore; + private final RemoteClusterStateBlobStore persistentSettingsBlobStore; + private final RemoteClusterStateBlobStore templatesMetadataBlobStore; + private final RemoteClusterStateBlobStore customMetadataBlobStore; + private final RemoteClusterStateBlobStore hashesOfConsistentSettingsBlobStore; + private final Compressor compressor; + private final NamedXContentRegistry namedXContentRegistry; - RemoteGlobalMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, BlobStoreTransferService blobStoreTransferService, - String clusterName) { - this.blobStoreRepository = blobStoreRepository; + RemoteGlobalMetadataManager(ClusterSettings clusterSettings, + RemoteWritableEntityStore globalMetadataBlobStore, + RemoteClusterStateBlobStore coordinationMetadataBlobStore, + RemoteClusterStateBlobStore transientSettingsBlobStore, + RemoteClusterStateBlobStore persistentSettingsBlobStore, + RemoteClusterStateBlobStore templatesMetadataBlobStore, + RemoteClusterStateBlobStore customMetadataBlobStore, + RemoteClusterStateBlobStore hashesOfConsistentSettingsBlobStore, + Compressor compressor, + NamedXContentRegistry namedXContentRegistry) { this.globalMetadataUploadTimeout = clusterSettings.get(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING); - this.threadPool = threadPool; - this.blobStoreTransferService = blobStoreTransferService; - this.clusterName = clusterName; - this.coordinationMetadataBlobStore = new RemoteCoordinationMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.transientSettingsBlobStore = new RemoteTransientSettingsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.persistentSettingsBlobStore = new RemotePersistentSettingsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.templatesMetadataBlobStore = new RemoteTemplatesMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.customMetadataBlobStore = new RemoteCustomMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.hashesOfConsistentSettingsBlobStore = new RemoteHashesOfConsistentSettingsBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.compressor = compressor; + this.globalMetadataBlobStore = globalMetadataBlobStore; + this.namedXContentRegistry = namedXContentRegistry; + this.coordinationMetadataBlobStore = coordinationMetadataBlobStore; + this.transientSettingsBlobStore = transientSettingsBlobStore; + this.persistentSettingsBlobStore = persistentSettingsBlobStore; + this.templatesMetadataBlobStore = templatesMetadataBlobStore; + this.customMetadataBlobStore = customMetadataBlobStore; + this.hashesOfConsistentSettingsBlobStore = hashesOfConsistentSettingsBlobStore; clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout); } @@ -139,39 +105,46 @@ CheckedRunnable getAsyncMetadataWriteAction( String customType ) { if (objectToUpload instanceof CoordinationMetadata) { - RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata((CoordinationMetadata) objectToUpload, metadataVersion, clusterUUID, - blobStoreRepository); - return () -> coordinationMetadataBlobStore.writeAsync(remoteCoordinationMetadata, getActionListener(remoteCoordinationMetadata, latchedActionListener)); - } else if (objectToUpload instanceof Settings) { + RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata((CoordinationMetadata) objectToUpload, metadataVersion, + clusterUUID, + compressor, namedXContentRegistry); + return () -> coordinationMetadataBlobStore.writeAsync(remoteCoordinationMetadata, + getActionListener(remoteCoordinationMetadata, latchedActionListener)); + } else if (objectToUpload instanceof Settings) { if (customType != null && customType.equals(TRANSIENT_SETTING_METADATA)) { - RemoteTransientSettingsMetadata remoteTransientSettingsMetadata = new RemoteTransientSettingsMetadata((Settings) objectToUpload, metadataVersion, clusterUUID, - blobStoreRepository); - return () -> transientSettingsBlobStore.writeAsync(remoteTransientSettingsMetadata, getActionListener(remoteTransientSettingsMetadata, latchedActionListener)); + RemoteTransientSettingsMetadata remoteTransientSettingsMetadata = new RemoteTransientSettingsMetadata((Settings) objectToUpload, + metadataVersion, clusterUUID, compressor, namedXContentRegistry); + return () -> transientSettingsBlobStore.writeAsync(remoteTransientSettingsMetadata, + getActionListener(remoteTransientSettingsMetadata, latchedActionListener)); } - RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata((Settings) objectToUpload, metadataVersion, clusterUUID, - blobStoreRepository); - return () -> persistentSettingsBlobStore.writeAsync(remotePersistentSettingsMetadata, getActionListener(remotePersistentSettingsMetadata, latchedActionListener)); + RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata((Settings) objectToUpload, metadataVersion, + clusterUUID, + compressor, namedXContentRegistry); + return () -> persistentSettingsBlobStore.writeAsync(remotePersistentSettingsMetadata, + getActionListener(remotePersistentSettingsMetadata, latchedActionListener)); } else if (objectToUpload instanceof TemplatesMetadata) { RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata((TemplatesMetadata) objectToUpload, metadataVersion, clusterUUID, - blobStoreRepository); + compressor, namedXContentRegistry); return () -> templatesMetadataBlobStore.writeAsync(remoteTemplatesMetadata, getActionListener(remoteTemplatesMetadata, latchedActionListener)); } else if (objectToUpload instanceof DiffableStringMap) { RemoteHashesOfConsistentSettings remoteObject = new RemoteHashesOfConsistentSettings( (DiffableStringMap) objectToUpload, metadataVersion, clusterUUID, - blobStoreRepository + compressor, + namedXContentRegistry ); return () -> hashesOfConsistentSettingsBlobStore.writeAsync(remoteObject, getActionListener(remoteObject, latchedActionListener)); } else if (objectToUpload instanceof Custom) { - RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata((Custom) objectToUpload, customType ,metadataVersion, clusterUUID, - blobStoreRepository); + RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata((Custom) objectToUpload, customType, metadataVersion, clusterUUID, + compressor, namedXContentRegistry); return () -> customMetadataBlobStore.writeAsync(remoteCustomMetadata, getActionListener(remoteCustomMetadata, latchedActionListener)); } throw new RemoteStateTransferException("Remote object cannot be created for " + objectToUpload.getClass()); } - private ActionListener getActionListener(AbstractRemoteBlobObject remoteBlobStoreObject, LatchedActionListener latchedActionListener) { + private ActionListener getActionListener(AbstractRemoteWritableBlobEntity remoteBlobStoreObject, + LatchedActionListener latchedActionListener) { return ActionListener.wrap( resp -> latchedActionListener.onResponse( remoteBlobStoreObject.getUploadedMetadata() @@ -187,41 +160,40 @@ CheckedRunnable getAsyncMetadataReadAction( String uploadFilename, LatchedActionListener listener ) { - ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); - if (component.equals(COORDINATION_METADATA)) { - RemoteCoordinationMetadata remoteBlobStoreObject = new RemoteCoordinationMetadata(uploadFilename, clusterUUID, blobStoreRepository); + ActionListener actionListener = ActionListener.wrap( + response -> listener.onResponse(new RemoteReadResult((ToXContent) response, component, componentName)), listener::onFailure); + if (component.equals(RemoteCoordinationMetadata.COORDINATION_METADATA)) { + RemoteCoordinationMetadata remoteBlobStoreObject = new RemoteCoordinationMetadata(uploadFilename, clusterUUID, compressor, namedXContentRegistry); return () -> coordinationMetadataBlobStore.readAsync(remoteBlobStoreObject, actionListener); - } else if (component.equals(TEMPLATES_METADATA)) { - RemoteTemplatesMetadata remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, blobStoreRepository); + } else if (component.equals(RemoteTemplatesMetadata.TEMPLATES_METADATA)) { + RemoteTemplatesMetadata remoteBlobStoreObject = new RemoteTemplatesMetadata(uploadFilename, clusterUUID, compressor, namedXContentRegistry); return () -> templatesMetadataBlobStore.readAsync(remoteBlobStoreObject, actionListener); - } else if (component.equals(SETTING_METADATA)) { - RemotePersistentSettingsMetadata remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, blobStoreRepository); + } else if (component.equals(RemotePersistentSettingsMetadata.SETTING_METADATA)) { + RemotePersistentSettingsMetadata remoteBlobStoreObject = new RemotePersistentSettingsMetadata(uploadFilename, clusterUUID, compressor, namedXContentRegistry); return () -> persistentSettingsBlobStore.readAsync(remoteBlobStoreObject, actionListener); - } else if (component.equals(TRANSIENT_SETTING_METADATA)) { - RemoteTransientSettingsMetadata remoteBlobStoreObject = new RemoteTransientSettingsMetadata(uploadFilename, clusterUUID, blobStoreRepository); + } else if (component.equals(RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA)) { + RemoteTransientSettingsMetadata remoteBlobStoreObject = new RemoteTransientSettingsMetadata(uploadFilename, clusterUUID, + compressor, namedXContentRegistry); return () -> transientSettingsBlobStore.readAsync(remoteBlobStoreObject, actionListener); - } else if (component.equals(CUSTOM_METADATA)) { - RemoteCustomMetadata remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, blobStoreRepository); + } else if (component.equals(RemoteCustomMetadata.CUSTOM_METADATA)) { + RemoteCustomMetadata remoteBlobStoreObject = new RemoteCustomMetadata(uploadFilename, componentName, clusterUUID, compressor, namedXContentRegistry); return () -> customMetadataBlobStore.readAsync(remoteBlobStoreObject, actionListener); } else if (component.equals(HASHES_OF_CONSISTENT_SETTINGS)) { - RemoteHashesOfConsistentSettings remoteHashesOfConsistentSettings = new RemoteHashesOfConsistentSettings(uploadFilename, clusterUUID, blobStoreRepository); + RemoteHashesOfConsistentSettings remoteHashesOfConsistentSettings = new RemoteHashesOfConsistentSettings(uploadFilename, clusterUUID, compressor, namedXContentRegistry); return () -> hashesOfConsistentSettingsBlobStore.readAsync(remoteHashesOfConsistentSettings, actionListener); } else { throw new RemoteStateTransferException("Unknown component " + componentName); } } - Metadata getGlobalMetadata(String clusterName, String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { + Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMetadataManifest) { String globalMetadataFileName = clusterMetadataManifest.getGlobalMetadataFileName(); try { // Fetch Global metadata if (globalMetadataFileName != null) { - String[] splitPath = globalMetadataFileName.split("/"); - return GLOBAL_METADATA_FORMAT.read( - globalMetadataContainer(clusterName, clusterUUID), - splitPath[splitPath.length - 1], - blobStoreRepository.getNamedXContentRegistry() - ); + RemoteGlobalMetadata remoteGlobalMetadata = new RemoteGlobalMetadata(globalMetadataFileName, clusterUUID, + compressor, namedXContentRegistry); + return globalMetadataBlobStore.read(remoteGlobalMetadata).get(); } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { CoordinationMetadata coordinationMetadata = getCoordinationMetadata( clusterUUID, @@ -266,8 +238,8 @@ public CoordinationMetadata getCoordinationMetadata(String clusterUUID, String c // Fetch Coordination metadata if (coordinationMetadataFileName != null) { RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(coordinationMetadataFileName, clusterUUID, - blobStoreRepository); - return coordinationMetadataBlobStore.read(remoteCoordinationMetadata); + compressor, namedXContentRegistry); + return coordinationMetadataBlobStore.read(remoteCoordinationMetadata).get(); } else { return CoordinationMetadata.EMPTY_METADATA; } @@ -284,8 +256,8 @@ public Settings getSettingsMetadata(String clusterUUID, String settingsMetadataF // Fetch Settings metadata if (settingsMetadataFileName != null) { RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(settingsMetadataFileName, clusterUUID, - blobStoreRepository); - return persistentSettingsBlobStore.read(remotePersistentSettingsMetadata); + compressor, namedXContentRegistry); + return persistentSettingsBlobStore.read(remotePersistentSettingsMetadata).get(); } else { return Settings.EMPTY; } @@ -302,8 +274,8 @@ public TemplatesMetadata getTemplatesMetadata(String clusterUUID, String templat // Fetch Templates metadata if (templatesMetadataFileName != null) { RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(templatesMetadataFileName, clusterUUID, - blobStoreRepository); - return templatesMetadataBlobStore.read(remoteTemplatesMetadata); + compressor, namedXContentRegistry); + return templatesMetadataBlobStore.read(remoteTemplatesMetadata).get(); } else { return TemplatesMetadata.EMPTY_METADATA; } @@ -319,8 +291,8 @@ public Metadata.Custom getCustomsMetadata(String clusterUUID, String customMetad requireNonNull(customMetadataFileName); try { // Fetch Custom metadata - RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, blobStoreRepository); - return customMetadataBlobStore.read(remoteCustomMetadata); + RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, compressor, namedXContentRegistry); + return customMetadataBlobStore.read(remoteCustomMetadata).get(); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), @@ -354,15 +326,9 @@ Map getUpdatedCustoms(ClusterState currentState, Cluste return updatedCustom; } - private BlobContainer globalMetadataContainer(String clusterName, String clusterUUID) { - // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/global-metadata/ - return blobStoreRepository.blobStore() - .blobContainer(getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(GLOBAL_METADATA_PATH_TOKEN)); - } - boolean isGlobalMetadataEqual(ClusterMetadataManifest first, ClusterMetadataManifest second, String clusterName) { - Metadata secondGlobalMetadata = getGlobalMetadata(clusterName, second.getClusterUUID(), second); - Metadata firstGlobalMetadata = getGlobalMetadata(clusterName, first.getClusterUUID(), first); + Metadata secondGlobalMetadata = getGlobalMetadata(second.getClusterUUID(), second); + Metadata firstGlobalMetadata = getGlobalMetadata(first.getClusterUUID(), first); return Metadata.isGlobalResourcesMetadataEquals(firstGlobalMetadata, secondGlobalMetadata); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index 67825546d54b2..77d6c0d200c24 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -20,12 +20,11 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.model.RemoteIndexMetadata; -import org.opensearch.gateway.remote.model.RemoteIndexMetadataBlobStore; import org.opensearch.gateway.remote.model.RemoteReadResult; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; +import org.opensearch.common.remote.RemoteWritableEntityStore; public class RemoteIndexMetadataManager { @@ -38,24 +37,18 @@ public class RemoteIndexMetadataManager { Setting.Property.NodeScope ); - private final BlobStoreRepository blobStoreRepository; - private final ThreadPool threadPool; - - private final BlobStoreTransferService blobStoreTransferService; - private final RemoteIndexMetadataBlobStore indexMetadataBlobStore; + private final RemoteWritableEntityStore indexMetadataBlobStore; + private final Compressor compressor; + private final NamedXContentRegistry namedXContentRegistry; private volatile TimeValue indexMetadataUploadTimeout; - private final String clusterName; - public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, ThreadPool threadPool, String clusterName, - BlobStoreTransferService blobStoreTransferService) { - indexMetadataBlobStore = new RemoteIndexMetadataBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - this.blobStoreRepository = blobStoreRepository; + public RemoteIndexMetadataManager(RemoteWritableEntityStore indexMetadataBlobStore, ClusterSettings clusterSettings, Compressor compressor, NamedXContentRegistry namedXContentRegistry) { + this.indexMetadataBlobStore = indexMetadataBlobStore; + this.compressor = compressor; + this.namedXContentRegistry = namedXContentRegistry; this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING); - this.threadPool = threadPool; clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout); - this.blobStoreTransferService = blobStoreTransferService; - this.clusterName = clusterName; } /** @@ -66,7 +59,7 @@ public RemoteIndexMetadataManager(BlobStoreRepository blobStoreRepository, Clust */ CheckedRunnable getIndexMetadataAsyncAction(IndexMetadata indexMetadata, String clusterUUID, LatchedActionListener latchedActionListener) { - RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); ActionListener completionListener = ActionListener.wrap( resp -> latchedActionListener.onResponse( remoteIndexMetadata.getUploadedMetadata() @@ -81,7 +74,7 @@ CheckedRunnable getAsyncIndexMetadataReadAction( String uploadedFilename, LatchedActionListener latchedActionListener ) { - RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); ActionListener actionListener = ActionListener.wrap( response -> latchedActionListener.onResponse(new RemoteReadResult(response, INDEX_PATH_TOKEN, response.getIndexName())), latchedActionListener::onFailure); @@ -98,9 +91,9 @@ IndexMetadata getIndexMetadata( ClusterMetadataManifest.UploadedIndexMetadata uploadedIndexMetadata, String clusterUUID, int manifestCodecVersion ) { RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(RemoteClusterStateUtils.getFormattedFileName( - uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, blobStoreRepository); + uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, compressor, namedXContentRegistry); try { - return indexMetadataBlobStore.read(remoteIndexMetadata); + return indexMetadataBlobStore.read(remoteIndexMetadata).get(); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading IndexMetadata - %s", uploadedIndexMetadata.getUploadedFilename()), diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index cf8ab3918e8ef..8126c6f774f8a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -8,6 +8,20 @@ package org.opensearch.gateway.remote; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.Version; @@ -20,28 +34,13 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; -import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifestBlobStore; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import org.opensearch.threadpool.ThreadPool; - -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; - public class RemoteManifestManager { public static final TimeValue METADATA_MANIFEST_UPLOAD_TIMEOUT_DEFAULT = TimeValue.timeValueMillis(20000); @@ -52,23 +51,24 @@ public class RemoteManifestManager { Setting.Property.Dynamic, Setting.Property.NodeScope ); + private static final Logger logger = LogManager.getLogger(RemoteManifestManager.class); - private final BlobStoreTransferService blobStoreTransferService; - private final BlobStoreRepository blobStoreRepository; private volatile TimeValue metadataManifestUploadTimeout; private final String nodeId; - private final ThreadPool threadPool; - private final RemoteClusterMetadataManifestBlobStore manifestBlobStore; - private static final Logger logger = LogManager.getLogger(RemoteManifestManager.class); + private final RemoteClusterStateBlobStore manifestBlobStore; + private final Compressor compressor; + private final NamedXContentRegistry namedXContentRegistry; + //todo remove blobStorerepo from here + private final BlobStoreRepository blobStoreRepository; - RemoteManifestManager(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, ClusterSettings clusterSettings, String nodeId, ThreadPool threadPool, String clusterName) { - this.blobStoreTransferService = blobStoreTransferService; - this.blobStoreRepository = blobStoreRepository; + RemoteManifestManager(RemoteClusterStateBlobStore manifestBlobStore, ClusterSettings clusterSettings, String nodeId, Compressor compressor, NamedXContentRegistry namedXContentRegistry, BlobStoreRepository blobStoreRepository) { this.metadataManifestUploadTimeout = clusterSettings.get(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING); this.nodeId = nodeId; - this.threadPool = threadPool; - manifestBlobStore = new RemoteClusterMetadataManifestBlobStore(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); + this.manifestBlobStore = manifestBlobStore; clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout); + this.compressor = compressor; + this.namedXContentRegistry = namedXContentRegistry; + this.blobStoreRepository = blobStoreRepository; } ClusterMetadataManifest uploadManifest( @@ -119,7 +119,7 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust logger.trace(String.format(Locale.ROOT, "Manifest file uploaded successfully.")); }, ex -> { exceptionReference.set(ex); }), latch); - RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(uploadManifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(uploadManifest, clusterUUID, compressor, namedXContentRegistry); manifestBlobStore.writeAsync(remoteClusterMetadataManifest, completionListener); try { @@ -183,8 +183,8 @@ ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, S throws IllegalStateException { try { String fullBlobName = getManifestFolderPath(clusterName, clusterUUID).buildAsString() + filename; - RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(fullBlobName, clusterUUID, blobStoreRepository); - return manifestBlobStore.read(remoteClusterMetadataManifest); + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(fullBlobName, clusterUUID, compressor, namedXContentRegistry); + return manifestBlobStore.read(remoteClusterMetadataManifest).get(); } catch (IOException e) { throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index f3906321301c9..f608fd0e352cc 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -17,17 +17,20 @@ import java.util.List; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading {@link ClusterBlocks} to/from remote blob store */ -public class RemoteClusterBlocks extends AbstractRemoteBlobObject { +public class RemoteClusterBlocks extends AbstractRemoteWritableBlobEntity { public static final String CLUSTER_BLOCKS = "blocks"; public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( @@ -39,15 +42,15 @@ public class RemoteClusterBlocks extends AbstractRemoteBlobObject private ClusterBlocks clusterBlocks; private long stateVersion; - public RemoteClusterBlocks(ClusterBlocks clusterBlocks, long stateVersion, String clusterUUID, - BlobStoreRepository blobStoreRepository) { - super(blobStoreRepository, clusterUUID); + public RemoteClusterBlocks(final ClusterBlocks clusterBlocks, long stateVersion, String clusterUUID, + final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); this.clusterBlocks = clusterBlocks; this.stateVersion = stateVersion; } - public RemoteClusterBlocks(String blobName, String clusterUUID, BlobStoreRepository blobStoreRepository) { - super(blobStoreRepository, clusterUUID); + public RemoteClusterBlocks(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; } @@ -76,6 +79,11 @@ public UploadedMetadata getUploadedMetadata() { return new UploadedMetadataAttribute(CLUSTER_BLOCKS, blobName); } + @Override + public void set(final ClusterBlocks clusterBlocks) { + this.clusterBlocks = clusterBlocks; + } + @Override public ClusterBlocks get() { return clusterBlocks; @@ -88,7 +96,7 @@ public InputStream serialize() throws IOException { } @Override - public ClusterBlocks deserialize(InputStream inputStream) throws IOException { - return CLUSTER_BLOCKS_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + public ClusterBlocks deserialize(final InputStream inputStream) throws IOException { + return CLUSTER_BLOCKS_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java deleted file mode 100644 index 31aede0b78dda..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksBlobStore.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.cluster.block.ClusterBlocks; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -/** - * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link ClusterBlocks} to/from blob store - */ -public class RemoteClusterBlocksBlobStore extends AbstractRemoteBlobStore { - - public RemoteClusterBlocksBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index 170836ce57baf..f55881ec8dc88 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -14,18 +14,21 @@ import java.io.InputStream; import java.util.List; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading {@link ClusterMetadataManifest} to/from remote blob store */ -public class RemoteClusterMetadataManifest extends AbstractRemoteBlobObject { +public class RemoteClusterMetadataManifest extends AbstractRemoteWritableBlobEntity { public static final String MANIFEST_PATH_TOKEN = "manifest"; public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; @@ -56,15 +59,15 @@ public class RemoteClusterMetadataManifest extends AbstractRemoteBlobObject blobStoreFormat = getClusterMetadataManifestBlobStoreFormat(); - return blobStoreFormat.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + return blobStoreFormat.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); } private int getManifestCodecVersion() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java deleted file mode 100644 index 09ed9046e9087..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestBlobStore.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.gateway.remote.ClusterMetadataManifest; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -/** - * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link ClusterMetadataManifest} to/from blob store - */ -public class RemoteClusterMetadataManifestBlobStore extends AbstractRemoteBlobStore{ - - public RemoteClusterMetadataManifestBlobStore(BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java similarity index 57% rename from server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java rename to server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java index 55822a195bd1d..62c706a18c1e8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/AbstractRemoteBlobStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java @@ -16,6 +16,9 @@ import java.util.concurrent.ExecutorService; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.stream.write.WritePriority; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.RemoteWritableEntityStore; +import org.opensearch.common.remote.RemoteWriteableEntity; import org.opensearch.core.action.ActionListener; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; @@ -26,55 +29,58 @@ * Abstract class for a blob type storage * * @param The entity which can be uploaded to / downloaded from blob store - * @param The concrete class implementing {@link RemoteObject} which is used as a wrapper for T entity. + * @param The concrete class implementing {@link RemoteWriteableEntity} which is used as a wrapper for T entity. */ -public class AbstractRemoteBlobStore> implements RemoteObjectStore { +public class RemoteClusterStateBlobStore> implements RemoteWritableEntityStore { private final BlobStoreTransferService transferService; private final BlobStoreRepository blobStoreRepository; private final String clusterName; private final ExecutorService executorService; - public AbstractRemoteBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { + public RemoteClusterStateBlobStore(final BlobStoreTransferService blobStoreTransferService, final BlobStoreRepository blobStoreRepository, final String clusterName, + final ThreadPool threadPool, final String executor) { this.transferService = blobStoreTransferService; this.blobStoreRepository = blobStoreRepository; this.clusterName = clusterName; - this.executorService = threadPool.executor(ThreadPool.Names.GENERIC); + this.executorService = threadPool.executor(executor); } @Override - public void writeAsync(U obj, ActionListener listener) { - assert obj.get() != null; + public void writeAsync(final U entity, final ActionListener listener) { + assert entity.get() != null; try { - InputStream inputStream = obj.serialize(); - BlobPath blobPath = getBlobPathForUpload(obj); - obj.setFullBlobName(blobPath); - transferService.uploadBlob(inputStream, getBlobPathForUpload(obj), obj.getBlobFileName(), WritePriority.URGENT, listener); + try (InputStream inputStream = entity.serialize()) { + BlobPath blobPath = getBlobPathForUpload(entity); + entity.setFullBlobName(blobPath); + transferService.uploadBlob(inputStream, getBlobPathForUpload(entity), entity.getBlobFileName(), WritePriority.URGENT, listener); + } } catch (Exception e) { listener.onFailure(e); } } @Override - public T read(U obj) throws IOException { - assert obj.getFullBlobName() != null; - return obj.deserialize( - transferService.downloadBlob(getBlobPathForDownload(obj), obj.getBlobFileName())); + public U read(final U entity) throws IOException { + assert entity.getFullBlobName() != null; + T object = entity.deserialize( + transferService.downloadBlob(getBlobPathForDownload(entity), entity.getBlobFileName())); + entity.set(object); + return entity; } @Override - public void readAsync(U obj, ActionListener listener) { + public void readAsync(final U entity, final ActionListener listener) { executorService.execute(() -> { try { - listener.onResponse(read(obj)); + listener.onResponse(read(entity).get()); } catch (Exception e) { listener.onFailure(e); } }); } - private BlobPath getBlobPathForUpload(AbstractRemoteBlobObject obj) { + private BlobPath getBlobPathForUpload(final AbstractRemoteWritableBlobEntity obj) { BlobPath blobPath = blobStoreRepository.basePath().add(RemoteClusterStateUtils.encodeString(clusterName)).add("cluster-state").add(obj.clusterUUID()); for (String token : obj.getBlobPathParameters().getPathTokens()) { blobPath = blobPath.add(token); @@ -82,7 +88,7 @@ private BlobPath getBlobPathForUpload(AbstractRemoteBlobObject obj) { return blobPath; } - private BlobPath getBlobPathForDownload(AbstractRemoteBlobObject obj) { + private BlobPath getBlobPathForDownload(final AbstractRemoteWritableBlobEntity obj) { String[] pathTokens = extractBlobPathTokens(obj.getFullBlobName()); BlobPath blobPath = new BlobPath(); for (String token : pathTokens) { @@ -91,7 +97,7 @@ private BlobPath getBlobPathForDownload(AbstractRemoteBlobObject obj) { return blobPath; } - private static String[] extractBlobPathTokens(String blobName) { + private static String[] extractBlobPathTokens(final String blobName) { String[] blobNameTokens = blobName.split(PATH_DELIMITER); return Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java index c248b2823246f..fffbcfab4e141 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -9,11 +9,15 @@ package org.opensearch.gateway.remote.model; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import java.io.IOException; @@ -26,7 +30,7 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; -public class RemoteClusterStateCustoms extends AbstractRemoteBlobObject { +public class RemoteClusterStateCustoms extends AbstractRemoteWritableBlobEntity { public static final String CLUSTER_STATE_CUSTOM = "cluster-state-custom"; public final ChecksumBlobStoreFormat clusterStateCustomBlobStoreFormat; @@ -34,8 +38,8 @@ public class RemoteClusterStateCustoms extends AbstractRemoteBlobObject( @@ -82,6 +86,11 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { return new ClusterMetadataManifest.UploadedMetadataAttribute(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType), blobName); } + @Override + public void set(Custom custom) { + this.custom = custom; + } + @Override public ClusterState.Custom get() { return custom; @@ -93,7 +102,7 @@ public InputStream serialize() throws IOException { } @Override - public ClusterState.Custom deserialize(InputStream inputStream) throws IOException { - return clusterStateCustomBlobStoreFormat.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + public ClusterState.Custom deserialize(final InputStream inputStream) throws IOException { + return clusterStateCustomBlobStoreFormat.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java deleted file mode 100644 index 2718347a05f47..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsBlobStore.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.cluster.ClusterState; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -public class RemoteClusterStateCustomsBlobStore extends AbstractRemoteBlobStore { - public RemoteClusterStateCustomsBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java index 76f97cd98c796..6b7368457fe89 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java @@ -17,17 +17,20 @@ import java.util.List; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading {@link CoordinationMetadata} to/from remote blob store */ -public class RemoteCoordinationMetadata extends AbstractRemoteBlobObject { +public class RemoteCoordinationMetadata extends AbstractRemoteWritableBlobEntity { public static final String COORDINATION_METADATA = "coordination"; public static final ChecksumBlobStoreFormat COORDINATION_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( @@ -39,14 +42,14 @@ public class RemoteCoordinationMetadata extends AbstractRemoteBlobObject { - - public RemoteCoordinationMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java index b5596af7ab1e9..e0d84955b5ea4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java @@ -19,17 +19,20 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading {@link Custom} to/from remote blob store */ -public class RemoteCustomMetadata extends AbstractRemoteBlobObject { +public class RemoteCustomMetadata extends AbstractRemoteWritableBlobEntity { public static final String CUSTOM_METADATA = "custom"; public static final String CUSTOM_DELIMITER = "--"; @@ -45,9 +48,9 @@ public class RemoteCustomMetadata extends AbstractRemoteBlobObject { private final String customType; private long metadataVersion; - public RemoteCustomMetadata(Custom custom, String customType, long metadataVersion, String clusterUUID, - BlobStoreRepository blobStoreRepository) { - super(blobStoreRepository, clusterUUID); + public RemoteCustomMetadata(final Custom custom, final String customType, final long metadataVersion, final String clusterUUID, + Compressor compressor, NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); this.custom = custom; this.customType = customType; this.metadataVersion = metadataVersion; @@ -58,9 +61,9 @@ public RemoteCustomMetadata(Custom custom, String customType, long metadataVersi ); } - public RemoteCustomMetadata(String blobName, String customType, String clusterUUID, - BlobStoreRepository blobStoreRepository) { - super(blobStoreRepository, clusterUUID); + public RemoteCustomMetadata(final String blobName, final String customType, final String clusterUUID, + final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; this.customType = customType; this.customBlobStoreFormat = new ChecksumBlobStoreFormat<>( @@ -91,6 +94,11 @@ public String generateBlobFileName() { return blobFileName; } + @Override + public void set(final Custom custom) { + this.custom = custom; + } + @Override public Custom get() { return custom; @@ -102,8 +110,8 @@ public InputStream serialize() throws IOException { } @Override - public Custom deserialize(InputStream inputStream) throws IOException { - return customBlobStoreFormat.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + public Custom deserialize(final InputStream inputStream) throws IOException { + return customBlobStoreFormat.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); } @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java deleted file mode 100644 index 1d24e014433d2..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataBlobStore.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.cluster.metadata.Metadata.Custom; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -/** - * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link Custom} to/from blob store - */ -public class RemoteCustomMetadataBlobStore extends AbstractRemoteBlobStore { - - public RemoteCustomMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index 07b272b5edfaa..34aca7767a0b1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -18,17 +18,20 @@ import java.util.List; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading {@link DiscoveryNodes} to/from remote blob store */ -public class RemoteDiscoveryNodes extends AbstractRemoteBlobObject { +public class RemoteDiscoveryNodes extends AbstractRemoteWritableBlobEntity { public static final String DISCOVERY_NODES = "nodes"; public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( @@ -40,14 +43,14 @@ public class RemoteDiscoveryNodes extends AbstractRemoteBlobObject { - - public RemoteDiscoveryNodesBlobStore(BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java new file mode 100644 index 0000000000000..42b8998bd6d32 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class RemoteGlobalMetadata extends AbstractRemoteWritableBlobEntity { + + public static final ChecksumBlobStoreFormat GLOBAL_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "metadata", + METADATA_NAME_FORMAT, + Metadata::fromXContent + ); + + private Metadata metadata; + private final String blobName; + + public RemoteGlobalMetadata(final String blobName, final String clusterUUID, final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + throw new UnsupportedOperationException(); + } + + @Override + public String generateBlobFileName() { + throw new UnsupportedOperationException(); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + throw new UnsupportedOperationException(); + } + + @Override + public void set(final Metadata metadata) { + this.metadata = metadata; + } + + @Override + public Metadata get() { + return metadata; + } + + @Override + public InputStream serialize() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Metadata deserialize(final InputStream inputStream) throws IOException { + return GLOBAL_METADATA_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java index 1cd9e48e07927..3dfa42139bb02 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -8,25 +8,26 @@ package org.opensearch.gateway.remote.model; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; - -public class RemoteHashesOfConsistentSettings extends AbstractRemoteBlobObject { +public class RemoteHashesOfConsistentSettings extends AbstractRemoteWritableBlobEntity { public static final String HASHES_OF_CONSISTENT_SETTINGS = "hashes-of-consistent-settings"; public static final ChecksumBlobStoreFormat HASHES_OF_CONSISTENT_SETTINGS_FORMAT = new ChecksumBlobStoreFormat<>( HASHES_OF_CONSISTENT_SETTINGS, @@ -36,14 +37,14 @@ public class RemoteHashesOfConsistentSettings extends AbstractRemoteBlobObject { - public RemoteHashesOfConsistentSettingsBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java index 983bc81f6c455..6f42488b79fc0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java @@ -16,17 +16,20 @@ import java.util.List; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading {@link IndexMetadata} to/from remote blob store */ -public class RemoteIndexMetadata extends AbstractRemoteBlobObject { +public class RemoteIndexMetadata extends AbstractRemoteWritableBlobEntity { public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; @@ -39,21 +42,28 @@ public class RemoteIndexMetadata extends AbstractRemoteBlobObject private IndexMetadata indexMetadata; - public RemoteIndexMetadata(IndexMetadata indexMetadata, String clusterUUID, BlobStoreRepository blobStoreRepository) { - super(blobStoreRepository, clusterUUID); + public RemoteIndexMetadata(final IndexMetadata indexMetadata, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); this.indexMetadata = indexMetadata; } - public RemoteIndexMetadata(String blobName, String clusterUUID, BlobStoreRepository blobStoreRepository) { - super(blobStoreRepository, clusterUUID); + public RemoteIndexMetadata(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; } + @Override + public void set(final IndexMetadata indexMetadata) { + this.indexMetadata = indexMetadata; + } + @Override public IndexMetadata get() { return indexMetadata; } + + @Override public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(INDEX_PATH_TOKEN, indexMetadata.getIndexUUID()), "metadata"); @@ -86,9 +96,9 @@ public InputStream serialize() throws IOException { } @Override - public IndexMetadata deserialize(InputStream inputStream) throws IOException { + public IndexMetadata deserialize(final InputStream inputStream) throws IOException { // Blob name parameter is redundant - return INDEX_METADATA_FORMAT.deserialize(blobName, getBlobStoreRepository().getNamedXContentRegistry(), Streams.readFully(inputStream)); + return INDEX_METADATA_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataBlobStore.java deleted file mode 100644 index 0a38147f9aa0b..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataBlobStore.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -/** - * An implementation of {@link RemoteObjectStore} which is used to upload/download {@link IndexMetadata} to/from blob store - */ -public class RemoteIndexMetadataBlobStore extends AbstractRemoteBlobStore { - public RemoteIndexMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, BlobStoreRepository blobStoreRepository, String clusterName, - ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } - -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java deleted file mode 100644 index ff63cd528fc2c..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsBlobStore.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.common.settings.Settings; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -/** - * An implementation of {@link RemoteObjectStore} which is used to upload/download persistent {@link Settings} to/from blob store - */ -public class RemotePersistentSettingsBlobStore extends AbstractRemoteBlobStore { - - public RemotePersistentSettingsBlobStore(BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java index 7dcbac745b49a..b2a8f07087cc5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java @@ -16,18 +16,21 @@ import java.io.InputStream; import java.util.List; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading persistent {@link Settings} to/from remote blob store */ -public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobObject { +public class RemotePersistentSettingsMetadata extends AbstractRemoteWritableBlobEntity { public static final String SETTING_METADATA = "settings"; @@ -40,14 +43,14 @@ public class RemotePersistentSettingsMetadata extends AbstractRemoteBlobObject { +public class RemoteTemplatesMetadata extends AbstractRemoteWritableBlobEntity { public static final String TEMPLATES_METADATA = "templates"; @@ -39,14 +42,14 @@ public class RemoteTemplatesMetadata extends AbstractRemoteBlobObject { - - public RemoteTemplatesMetadataBlobStore(BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java deleted file mode 100644 index 5004cef8a9c66..0000000000000 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsBlobStore.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.gateway.remote.model; - -import org.opensearch.common.settings.Settings; -import org.opensearch.index.translog.transfer.BlobStoreTransferService; -import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.threadpool.ThreadPool; - -/** - * An implementation of {@link RemoteObjectStore} which is used to upload/download transient {@link Settings} to/from blob store - */ -public class RemoteTransientSettingsBlobStore extends AbstractRemoteBlobStore { - - public RemoteTransientSettingsBlobStore(BlobStoreTransferService blobStoreTransferService, - BlobStoreRepository blobStoreRepository, String clusterName, ThreadPool threadPool) { - super(blobStoreTransferService, blobStoreRepository, clusterName, threadPool); - } -} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java index ee0116a54bbae..75b6cfc3a765a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java @@ -16,18 +16,21 @@ import java.io.InputStream; import java.util.List; import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; /** * Wrapper class for uploading/downloading transient {@link Settings} to/from remote blob store */ -public class RemoteTransientSettingsMetadata extends AbstractRemoteBlobObject { +public class RemoteTransientSettingsMetadata extends AbstractRemoteWritableBlobEntity { public static final String TRANSIENT_SETTING_METADATA = "transient-settings"; @@ -40,14 +43,14 @@ public class RemoteTransientSettingsMetadata extends AbstractRemoteBlobObject= ClusterMetadataManifest.CODEC_V2) { String coordinationFileName = getFileNameFromPath(clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename()); - when(blobContainer.readBlob(RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))) + when(blobContainer.readBlob(RemoteCoordinationMetadata.COORDINATION_METADATA_FORMAT.blobName(coordinationFileName))) .thenAnswer((invocationOnMock) -> { - BytesReference bytesReference = RemoteGlobalMetadataManager.COORDINATION_METADATA_FORMAT.serialize( + BytesReference bytesReference = RemoteCoordinationMetadata.COORDINATION_METADATA_FORMAT.serialize( metadata.coordinationMetadata(), coordinationFileName, blobStoreRepository.getCompressor(), @@ -1617,9 +1621,9 @@ private void mockBlobContainerForGlobalMetadata( }); String settingsFileName = getFileNameFromPath(clusterMetadataManifest.getSettingsMetadata().getUploadedFilename()); - when(blobContainer.readBlob(RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT.blobName(settingsFileName))).thenAnswer( + when(blobContainer.readBlob(RemotePersistentSettingsMetadata.SETTINGS_METADATA_FORMAT.blobName(settingsFileName))).thenAnswer( (invocationOnMock) -> { - BytesReference bytesReference = RemoteGlobalMetadataManager.SETTINGS_METADATA_FORMAT.serialize( + BytesReference bytesReference = RemotePersistentSettingsMetadata.SETTINGS_METADATA_FORMAT.serialize( metadata.persistentSettings(), settingsFileName, blobStoreRepository.getCompressor(), @@ -1630,9 +1634,9 @@ private void mockBlobContainerForGlobalMetadata( ); String templatesFileName = getFileNameFromPath(clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename()); - when(blobContainer.readBlob(RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT.blobName(templatesFileName))).thenAnswer( + when(blobContainer.readBlob(RemoteTemplatesMetadata.TEMPLATES_METADATA_FORMAT.blobName(templatesFileName))).thenAnswer( (invocationOnMock) -> { - BytesReference bytesReference = RemoteGlobalMetadataManager.TEMPLATES_METADATA_FORMAT.serialize( + BytesReference bytesReference = RemoteTemplatesMetadata.TEMPLATES_METADATA_FORMAT.serialize( metadata.templatesMetadata(), templatesFileName, blobStoreRepository.getCompressor(), @@ -1664,9 +1668,9 @@ private void mockBlobContainerForGlobalMetadata( } } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); - when(blobContainer.readBlob(RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))) + when(blobContainer.readBlob(RemoteGlobalMetadata.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))) .thenAnswer((invocationOnMock) -> { - BytesReference bytesGlobalMetadata = RemoteGlobalMetadataManager.GLOBAL_METADATA_FORMAT.serialize( + BytesReference bytesGlobalMetadata = RemoteGlobalMetadata.GLOBAL_METADATA_FORMAT.serialize( metadata, "global-metadata-file", blobStoreRepository.getCompressor(), diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java index b76f8c3dbf9da..9ab8b7f340458 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManagerTests.java @@ -8,19 +8,42 @@ package org.opensearch.gateway.remote; +import static java.util.stream.Collectors.toList; +import static org.mockito.Mockito.mock; + +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.Metadata.Custom; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.model.RemoteCoordinationMetadata; +import org.opensearch.gateway.remote.model.RemoteCustomMetadata; +import org.opensearch.gateway.remote.model.RemoteGlobalMetadata; +import org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings; +import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata; +import org.opensearch.gateway.remote.model.RemoteTemplatesMetadata; +import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; -import org.junit.After; -import org.junit.Before; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; -import static org.mockito.Mockito.mock; - public class RemoteGlobalMetadataManagerTests extends OpenSearchTestCase { + + private static final String CLUSTER_NAME = "test-cluster"; private RemoteGlobalMetadataManager remoteGlobalMetadataManager; private ClusterSettings clusterSettings; private BlobStoreRepository blobStoreRepository; @@ -31,7 +54,34 @@ public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); BlobStoreTransferService blobStoreTransferService = mock(BlobStoreTransferService.class); - remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(blobStoreRepository, clusterSettings, threadPool, blobStoreTransferService, "test-cluster"); + RemoteClusterStateBlobStore globalMetadataBlobStore = new RemoteClusterStateBlobStore<>( + blobStoreTransferService, blobStoreRepository, CLUSTER_NAME, threadPool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore coordinationMetadataBlobStore = new RemoteClusterStateBlobStore<>( + blobStoreTransferService, blobStoreRepository, CLUSTER_NAME, threadPool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore transientSettingsBlobStore = new RemoteClusterStateBlobStore<>( + blobStoreTransferService, blobStoreRepository, CLUSTER_NAME, threadPool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore persistentSettingsBlobStore = new RemoteClusterStateBlobStore<>( + blobStoreTransferService, blobStoreRepository, CLUSTER_NAME, threadPool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore templatesMetadataBlobStore = new RemoteClusterStateBlobStore<>( + blobStoreTransferService, blobStoreRepository, CLUSTER_NAME, threadPool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore customMetadataBlobStore = new RemoteClusterStateBlobStore<>(blobStoreTransferService, + blobStoreRepository, CLUSTER_NAME, threadPool, ThreadPool.Names.GENERIC); + RemoteClusterStateBlobStore hashesOfConsistentSettingsBlobStore = + new RemoteClusterStateBlobStore<>( + blobStoreTransferService, + blobStoreRepository, CLUSTER_NAME, threadPool, ThreadPool.Names.GENERIC); + NamedXContentRegistry xContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + remoteGlobalMetadataManager = new RemoteGlobalMetadataManager(clusterSettings, globalMetadataBlobStore, coordinationMetadataBlobStore, + transientSettingsBlobStore, + persistentSettingsBlobStore, templatesMetadataBlobStore, customMetadataBlobStore, + hashesOfConsistentSettingsBlobStore, + new NoneCompressor(), xContentRegistry); } @After diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java index 87c19e883fa2f..e3ba051f3b1da 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteIndexMetadataManagerTests.java @@ -8,16 +8,28 @@ package org.opensearch.gateway.remote; +import static java.util.stream.Collectors.toList; +import static org.mockito.Mockito.mock; + +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.model.RemoteIndexMetadata; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; -import org.junit.After; -import org.junit.Before; import org.opensearch.threadpool.TestThreadPool; - -import static org.mockito.Mockito.mock; +import org.opensearch.threadpool.ThreadPool; public class RemoteIndexMetadataManagerTests extends OpenSearchTestCase { private RemoteIndexMetadataManager remoteIndexMetadataManager; @@ -30,7 +42,15 @@ public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); blobStoreTransferService = mock(BlobStoreTransferService.class); - remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStoreRepository, clusterSettings, new TestThreadPool("test"),"cluster-name", blobStoreTransferService); + RemoteClusterStateBlobStore blobStore = new RemoteClusterStateBlobStore<>(blobStoreTransferService, blobStoreRepository, "cluster-name", new TestThreadPool("test"), ThreadPool.Names.GENERIC); + NamedXContentRegistry xContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + remoteIndexMetadataManager = new RemoteIndexMetadataManager(blobStore, clusterSettings, new NoneCompressor(), xContentRegistry); } @After diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java index d1a4b89fe4379..79ec8c4b13ff7 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteManifestManagerTests.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote; +import static java.util.stream.Collectors.toList; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -17,18 +18,28 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import java.io.IOException; +import java.util.function.Function; +import java.util.stream.Stream; import org.junit.After; import org.junit.Before; +import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; public class RemoteManifestManagerTests extends OpenSearchTestCase { private RemoteManifestManager remoteManifestManager; @@ -41,8 +52,16 @@ public class RemoteManifestManagerTests extends OpenSearchTestCase { public void setup() { clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); blobStoreRepository = mock(BlobStoreRepository.class); - remoteManifestManager = new RemoteManifestManager(blobStoreTransferService, blobStoreRepository, clusterSettings, "test-node-id", new TestThreadPool("test"), "test-cluster-name"); + NamedXContentRegistry xContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); blobStoreTransferService = mock(BlobStoreTransferService.class); + RemoteClusterStateBlobStore manifestBlobStore = new RemoteClusterStateBlobStore<>(blobStoreTransferService, blobStoreRepository, "test-cluster-name", new TestThreadPool("test"), ThreadPool.Names.GENERIC); + remoteManifestManager = new RemoteManifestManager(manifestBlobStore, clusterSettings, "test-node-id", new NoneCompressor(), xContentRegistry, blobStoreRepository); blobStore = mock(BlobStore.class); when(blobStoreRepository.blobStore()).thenReturn(blobStore); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java index 58c5058e95e74..24d13a3b7a7fd 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTest.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote.model; +import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -21,20 +22,29 @@ import java.io.InputStream; import java.util.EnumSet; import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; import org.junit.After; import org.junit.Before; +import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.block.ClusterBlock; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -53,6 +63,8 @@ public class RemoteClusterBlocksTest extends OpenSearchTestCase { private BlobStoreRepository blobStoreRepository; private String clusterName; private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @Before @@ -65,6 +77,14 @@ public void setup() { when(blobStoreRepository.basePath()).thenReturn(blobPath); when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); this.clusterName = "test-cluster-name"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); } @After @@ -75,43 +95,43 @@ public void tearDown() throws Exception { public void testGet() { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.get(), is(clusterBlocks)); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.get(), nullValue()); } public void testClusterUUID() { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); } public void testFullBlobName() { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); } public void testBlobFileName() { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); } public void testBlobPathParameters() { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertThat(params.getPathTokens(), is(List.of(PATH_PARAM_TRANSIENT))); assertThat(params.getFilePrefix(), is(CLUSTER_BLOCKS)); @@ -119,7 +139,7 @@ public void testBlobPathParameters() { public void testGenerateBlobFileName() { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); assertThat(nameTokens[0], is(CLUSTER_BLOCKS)); @@ -130,7 +150,7 @@ public void testGenerateBlobFileName() { public void testGetUploadedMetadata() throws IOException { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); try (InputStream inputStream = remoteObjectForUpload.serialize()) { @@ -143,7 +163,7 @@ public void testGetUploadedMetadata() throws IOException { public void testSerDe() throws IOException { ClusterBlocks clusterBlocks = getClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, blobStoreRepository); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, VERSION, clusterUUID, compressor, namedXContentRegistry); try (InputStream inputStream = remoteObjectForUpload.serialize()) { assertThat(inputStream.available(), greaterThan(0)); ClusterBlocks readClusterBlocks = remoteObjectForUpload.deserialize(inputStream); diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java index 1e71c3c0dab0c..f50d90e0f6cf3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote.model; +import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -22,13 +23,21 @@ import java.io.InputStream; import java.util.Collections; import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; import org.junit.After; import org.junit.Before; import org.opensearch.Version; +import org.opensearch.cluster.ClusterModule; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; @@ -36,6 +45,7 @@ import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -55,6 +65,8 @@ public class RemoteClusterMetadataManifestTests extends OpenSearchTestCase { private BlobStoreRepository blobStoreRepository; private String clusterName; private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @Before @@ -66,6 +78,14 @@ public void setup() { BlobPath blobPath = new BlobPath().add("/path"); when(blobStoreRepository.basePath()).thenReturn(blobPath); when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); this.clusterName = "test-cluster-name"; } @@ -77,43 +97,43 @@ public void tearDown() throws Exception { public void testGet() { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.get(), is(manifest)); - RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.get(), nullValue()); } public void testClusterUUID() { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); - RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); } public void testFullBlobName() { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); - RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); } public void testBlobFileName() { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); - RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); } public void testBlobPathParameters() { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertThat(params.getPathTokens(), is(List.of(MANIFEST_PATH_TOKEN))); assertThat(params.getFilePrefix(), is(MANIFEST_FILE_PREFIX)); @@ -121,7 +141,7 @@ public void testBlobPathParameters() { public void testGenerateBlobFileName() { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); assertThat(nameTokens[0], is(MANIFEST_FILE_PREFIX)); @@ -134,7 +154,7 @@ public void testGenerateBlobFileName() { public void testGetUploadedMetadata() throws IOException { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); try (InputStream inputStream = remoteObjectForUpload.serialize()) { @@ -147,7 +167,7 @@ public void testGetUploadedMetadata() throws IOException { public void testSerDe() throws IOException { ClusterMetadataManifest manifest = getClusterMetadataManifest(); - RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, blobStoreRepository); + RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); try (InputStream inputStream = remoteObjectForUpload.serialize()) { remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); assertThat(inputStream.available(), greaterThan(0)); diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java index b6a4d0ba60ebe..db0e9d9f41e88 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteIndexMetadataTests.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote.model; +import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -21,21 +22,28 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; import org.junit.After; import org.junit.Before; import org.opensearch.Version; +import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.RemoteClusterStateUtils; -import org.opensearch.gateway.remote.model.BlobPathParameters; -import org.opensearch.gateway.remote.model.RemoteIndexMetadata; +import org.opensearch.common.remote.BlobPathParameters; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; @@ -53,6 +61,8 @@ public class RemoteIndexMetadataTests extends OpenSearchTestCase { private BlobStoreRepository blobStoreRepository; private String clusterName; private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @Before @@ -65,6 +75,14 @@ public void setup() { when(blobStoreRepository.basePath()).thenReturn(blobPath); when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); this.clusterName = "test-cluster-name"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); } @After @@ -75,43 +93,43 @@ public void tearDown() throws Exception { public void testGet() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.get(), is(indexMetadata)); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.get(), nullValue()); } public void testClusterUUID() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); } public void testFullBlobName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); } public void testBlobFileName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); - RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForDownload = new RemoteIndexMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); } public void testBlobPathParameters() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertThat(params.getPathTokens(), is(List.of(INDEX_PATH_TOKEN, indexMetadata.getIndexUUID()))); assertThat(params.getFilePrefix(), is("metadata")); @@ -119,7 +137,7 @@ public void testBlobPathParameters() { public void testGenerateBlobFileName() { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); assertThat(nameTokens[0], is("metadata")); @@ -130,7 +148,7 @@ public void testGenerateBlobFileName() { public void testGetUploadedMetadata() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); try (InputStream inputStream = remoteObjectForUpload.serialize()){ @@ -143,7 +161,7 @@ public void testGetUploadedMetadata() throws IOException { public void testSerDe() throws IOException { IndexMetadata indexMetadata = getIndexMetadata(); - RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, blobStoreRepository); + RemoteIndexMetadata remoteObjectForUpload = new RemoteIndexMetadata(indexMetadata, clusterUUID, compressor, namedXContentRegistry); try (InputStream inputStream = remoteObjectForUpload.serialize()) { assertThat(inputStream.available(), greaterThan(0)); IndexMetadata readIndexMetadata = remoteObjectForUpload.deserialize(inputStream); From 6776c0304dd81d835e8c0fe49dd09389ebd34c6e Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Mon, 3 Jun 2024 15:42:54 +0530 Subject: [PATCH 109/133] Implement diffUtils in RemoteRoutingTable Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 65 ++++++++----------- .../remote/ClusterStateDiffManifest.java | 11 +++- .../remote/RemoteClusterStateService.java | 7 +- 3 files changed, 41 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 046b56191d9f1..8dbcc7a5bad9f 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -13,6 +13,8 @@ import org.apache.lucene.store.IndexInput; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.Diff; +import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.CheckedRunnable; @@ -28,6 +30,8 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.index.Index; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateUtils; @@ -88,6 +92,19 @@ public class RemoteRoutingTableService implements Closeable { private BlobStoreRepository blobStoreRepository; private final ThreadPool threadPool; + private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = new DiffableUtils.NonDiffableValueSerializer() { + @Override + public void write(IndexRoutingTable value, StreamOutput out) throws IOException { + value.writeTo(out); + } + + @Override + public IndexRoutingTable read(StreamInput in, String key) throws IOException { + return IndexRoutingTable.readFrom(in); + } + }; + + public RemoteRoutingTableService(Supplier repositoriesService, Settings settings, ThreadPool threadPool) { @@ -97,19 +114,6 @@ public RemoteRoutingTableService(Supplier repositoriesServi this.threadPool = threadPool; } - public List getChangedIndicesRouting( ClusterState previousClusterState, - ClusterState clusterState) { - Map previousIndexRoutingTable = previousClusterState.getRoutingTable().getIndicesRouting(); - List changedIndicesRouting = new ArrayList<>(); - for (IndexRoutingTable indexRouting : clusterState.getRoutingTable().getIndicesRouting().values()) { - if (!(previousIndexRoutingTable.containsKey(indexRouting.getIndex().getName()) && indexRouting.equals(previousIndexRoutingTable.get(indexRouting.getIndex().getName())))) { - changedIndicesRouting.add(indexRouting); - logger.info("changedIndicesRouting {}", indexRouting.prettyPrint()); - } - } - - return changedIndicesRouting; - } public CheckedRunnable getIndexRoutingAsyncAction( ClusterState clusterState, @@ -202,7 +206,7 @@ public List getAllUploadedIndices uploadedIndexRouting -> allUploadedIndicesRouting.put(uploadedIndexRouting.getIndexName(), uploadedIndexRouting) ); - indicesRoutingToDelete.forEach(index -> allUploadedIndicesRouting.remove(index)); + indicesRoutingToDelete.forEach(allUploadedIndicesRouting::remove); logger.info("allUploadedIndicesRouting ROUTING {}", allUploadedIndicesRouting); @@ -274,34 +278,17 @@ public List getUpdatedIndexRoutin }).collect(Collectors.toList()); } - public static List getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List deletedIndicesRouting = new ArrayList<>(); - for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { - if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is deleted - deletedIndicesRouting.add(previousIndexRouting.getIndex().getName()); - } - } - return deletedIndicesRouting; - } - public static List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List updatedIndicesRouting = new ArrayList<>(); - for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { - if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is created - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } else { - if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { - // if the latest routing table has the same routing table as the previous routing table, then the index is not updated - continue; - } - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } - } - return updatedIndicesRouting; + public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after) { + return DiffableUtils.diff( + before.getIndicesRouting(), + after.getIndicesRouting(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER + ); } + @Override public void close() throws IOException { if (blobStoreRepository != null) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 2ad61c8da2a5f..35263adc540e2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -10,6 +10,7 @@ import org.apache.lucene.util.packed.DirectMonotonicReader; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; @@ -89,8 +90,14 @@ public class ClusterStateDiffManifest implements ToXContentObject { customMetadataDeleted.add(custom); } } - indicesRoutingUpdated = RemoteRoutingTableService.getIndicesRoutingUpdated(previousState.routingTable(), state.routingTable()); - indicesRoutingDeleted = RemoteRoutingTableService.getIndicesRoutingDeleted(previousState.routingTable(), state.routingTable()); + + DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff(previousState.getRoutingTable(), + state.getRoutingTable()); + + indicesRoutingUpdated = new ArrayList<>(); + routingTableDiff.getUpserts().forEach((k,v) -> indicesRoutingUpdated.add(k)); + + indicesRoutingDeleted = routingTableDiff.getDeletes(); hashesOfConsistentSettingsUpdated = !state.metadata().hashesOfConsistentSettings().equals(previousState.metadata().hashesOfConsistentSettings()); clusterStateCustomUpdated = new ArrayList<>(); clusterStateCustomDeleted = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 32e8761691129..7f5828eee7b29 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -51,6 +51,8 @@ import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.Diff; +import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.coordination.CoordinationMetadata; import org.opensearch.cluster.metadata.DiffableStringMap; @@ -337,9 +339,12 @@ public ClusterMetadataManifest writeIncrementalMetadata( } List indicesRoutingToUpload = new ArrayList<>(); + DiffableUtils.MapDiff> routingTableDiff = null; if (remoteRoutingTableService != null) { - indicesRoutingToUpload = remoteRoutingTableService.getChangedIndicesRouting(previousClusterState, clusterState); + routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff(previousClusterState.getRoutingTable(), clusterState.getRoutingTable()); + routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingToUpload.add(v)); } + UploadedMetadataResults uploadedMetadataResults; // For migration case from codec V0 or V1 to V2, we have added null check on metadata attribute files, // If file is empty and codec is 1 then write global metadata. From 65a04a2d7a4fbf43c04803ecd455e315bf7d85ec Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Mon, 3 Jun 2024 16:10:23 +0530 Subject: [PATCH 110/133] refactoring in remote routing table service Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 23 ++----------------- .../remote/RemoteClusterStateService.java | 8 +++---- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 8dbcc7a5bad9f..81cf94f640163 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -237,7 +237,7 @@ private String getIndexRoutingFileName() { public CheckedRunnable getAsyncIndexMetadataReadAction( String uploadedFilename, Index index, - LatchedActionListener latchedActionListener) { + LatchedActionListener latchedActionListener) { int idx = uploadedFilename.lastIndexOf("/"); String blobFileName = uploadedFilename.substring(idx+1); BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer( BlobPath.cleanPath().add(uploadedFilename.substring(0,idx))); @@ -246,7 +246,7 @@ public CheckedRunnable getAsyncIndexMetadataReadAction( blobContainer, blobFileName, threadPool.executor(ThreadPool.Names.GENERIC), - ActionListener.wrap(response -> latchedActionListener.onResponse(new RemoteIndexRoutingResult(index.getName(), response.readIndexRoutingTable(index))), latchedActionListener::onFailure) + ActionListener.wrap(response -> latchedActionListener.onResponse(response.readIndexRoutingTable(index)), latchedActionListener::onFailure) ); } @@ -255,7 +255,6 @@ public void readAsync(BlobContainer blobContainer, String name, ExecutorService try { listener.onResponse(read(blobContainer, name)); } catch (Exception e) { - logger.error("routing table download failed : ", e); listener.onFailure(e); } }); @@ -307,22 +306,4 @@ public void start() { blobStoreRepository = (BlobStoreRepository) repository; } - public static class RemoteIndexRoutingResult { - String indexName; - IndexRoutingTable indexRoutingTable; - - public RemoteIndexRoutingResult(String indexName, IndexRoutingTable indexRoutingTable) { - this.indexName = indexName; - this.indexRoutingTable = indexRoutingTable; - } - - public String getIndexName() { - return indexName; - } - - public IndexRoutingTable getIndexRoutingTable() { - return indexRoutingTable; - } - } - } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 7f5828eee7b29..c835fe83b3e06 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -985,7 +985,7 @@ private ClusterState readClusterStateInParallel( CountDownLatch latch = new CountDownLatch(totalReadTasks); List> asyncMetadataReadActions = new ArrayList<>(); List readResults = new ArrayList<>(); - List readIndexRoutingTableResults = new ArrayList<>(); + List readIndexRoutingTableResults = new ArrayList<>(); List exceptionList = Collections.synchronizedList(new ArrayList<>(totalReadTasks)); LatchedActionListener listener = new LatchedActionListener<>( @@ -1012,7 +1012,7 @@ private ClusterState readClusterStateInParallel( ); } - LatchedActionListener routingTableLatchedActionListener = new LatchedActionListener<>( + LatchedActionListener routingTableLatchedActionListener = new LatchedActionListener<>( ActionListener.wrap( response -> { logger.debug("Successfully read cluster state component from remote"); @@ -1187,8 +1187,8 @@ private ClusterState readClusterStateInParallel( } }); - readIndexRoutingTableResults.forEach(remoteIndexRoutingResult -> { - indicesRouting.put(remoteIndexRoutingResult.getIndexName(), remoteIndexRoutingResult.getIndexRoutingTable()); + readIndexRoutingTableResults.forEach(indexRoutingTable -> { + indicesRouting.put(indexRoutingTable.getIndex().getName(), indexRoutingTable); }); metadataBuilder.indices(indexMetadataMap); From 2afe186e899b039553fd9399f48d0aa647bcc60b Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 4 Jun 2024 14:25:53 +0530 Subject: [PATCH 111/133] Fix snapshot de/ser Signed-off-by: Shivansh Arora --- .../cluster/SnapshotDeletionsInProgress.java | 13 +++- .../cluster/SnapshotsInProgress.java | 75 +++++++++++++++---- .../org/opensearch/repositories/IndexId.java | 2 +- ...SnapshotsInProgressSerializationTests.java | 52 ++++++++++++- 4 files changed, 122 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index fed4332e99361..b59e0d8214932 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -209,6 +209,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repository_state_id", entry.repositoryStateId); if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { builder.field("state", entry.state().value); + builder.field("uuid", entry.uuid()); } // else we don't serialize it } builder.endObject(); @@ -218,6 +219,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } ensureFieldName(parser, parser.currentToken(), TYPE); parser.nextToken(); ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); @@ -229,6 +236,7 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th byte stateValue = -1; List snapshotIds = new ArrayList<>(); TimeValue startTime = null; + String entryUUID = null; while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); final String fieldName = parser.currentName(); @@ -270,12 +278,15 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th case "state": stateValue = (byte) parser.intValue(); break; + case "uuid": + entryUUID = parser.text(); + break; default: throw new IllegalArgumentException("unknown field [" + fieldName + "]"); } } assert startTime != null; - entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue))); + entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue), entryUUID)); } return SnapshotDeletionsInProgress.of(entries); } diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index 700b25d3d1e54..9a3f582995814 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -66,6 +67,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.common.xcontent.XContentUtils.readValue; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; @@ -740,7 +742,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(STATE, status.state()); builder.field(NODE, status.nodeId()); if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { - builder.field(INDEX_UUID, shardId.getIndex().getUUID()); if (status.generation() != null) builder.field(GENERATION, status.generation()); if (status.reason() != null) builder.field(REASON, status.reason()); } @@ -770,6 +771,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (status.reason() != null) builder.field(REASON, status.reason()); builder.endObject(); } + builder.endArray(); builder.field(REMOTE_STORE_INDEX_SHALLOW_COPY, remoteStoreIndexShallowCopy); } builder.endObject(); @@ -786,7 +788,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { Version version = null; SnapshotId source = null; Map metadata = null; - byte state = -1; + State state = null; List indices = new ArrayList<>(); long startTime = 0; long repositoryStateId = -1L; @@ -794,11 +796,10 @@ public static Entry fromXContent(XContentParser parser) throws IOException { List dataStreams = new ArrayList<>(); Map clones = new HashMap<>(); boolean remoteStoreIndexShallowCopy = false; - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { String currentFieldName = parser.currentName(); parser.nextToken(); - switch (currentFieldName) { case REPOSITORY: repository = parser.text(); @@ -816,7 +817,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { partial = parser.booleanValue(); break; case STATE: - state = (byte) parser.intValue(); + state = State.fromString(parser.text()); break; case INDICES: ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); @@ -836,32 +837,28 @@ public static Entry fromXContent(XContentParser parser) throws IOException { case SHARDS: ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - String index = null; - String indexUUID = null; + Index index = null; int shardId = -1; String nodeId = null; ShardState shardState = null; String reason = null; String generation = null; - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { final String currentShardField = parser.currentName(); parser.nextToken(); switch (currentShardField) { case INDEX: - index = parser.text(); + index = Index.fromXContent(parser); break; case SHARD: shardId = parser.intValue(); break; - case INDEX_UUID: - indexUUID = parser.text(); - break; case NODE: - nodeId = parser.text(); + nodeId = (String) readValue(parser, parser.currentToken()); break; case STATE: - shardState = ShardState.fromValue((byte) parser.intValue()); + shardState = ShardState.fromString(parser.text()); break; case REASON: reason = parser.text(); @@ -873,7 +870,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { throw new IllegalArgumentException("unknown field [" + currentShardField + "]"); } } - shards.put(new ShardId(index, indexUUID, shardId), + shards.put(new ShardId(index, shardId), reason != null ? new ShardSnapshotStatus(nodeId, shardState, reason, generation) : new ShardSnapshotStatus(nodeId, shardState, generation)); } @@ -950,7 +947,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { snapshot, includeGlobalState, partial, - State.fromValue(state), + state, indices, dataStreams, startTime, @@ -1203,6 +1200,25 @@ public static State fromValue(byte value) { throw new IllegalArgumentException("No snapshot state for value [" + value + "]"); } } + + public static State fromString(String value) { + switch(value) { + case "INIT": + return INIT; + case "STARTED": + return STARTED; + case "SUCCESS": + return SUCCESS; + case "FAILED": + return FAILED; + case "ABORTED": + return ABORTED; + case "PARTIAL": + return PARTIAL; + default: + throw new IllegalArgumentException("No snapshot state for value [" + value + "]"); + } + } } private final List entries; @@ -1311,6 +1327,12 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } public static SnapshotsInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } ensureFieldName(parser, parser.currentToken(), SNAPSHOTS); ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser); List entries = new ArrayList<>(); @@ -1380,5 +1402,26 @@ public static ShardState fromValue(byte value) { throw new IllegalArgumentException("No shard snapshot state for value [" + value + "]"); } } + + public static ShardState fromString(String state) { + switch (state) { + case "INIT": + return INIT; + case "SUCCESS": + return SUCCESS; + case "FAILED": + return FAILED; + case "ABORTED": + return ABORTED; + case "MISSING": + return MISSING; + case "WAITING": + return WAITING; + case "QUEUED": + return QUEUED; + default: + throw new IllegalArgumentException("No shard snapshot state for value [" + state + "]"); + } + } } } diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index 1a2e797c62847..2f97368d42038 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -138,7 +138,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } public static IndexId fromXContent(XContentParser parser) throws IOException { - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); String name = null; String id = null; while (parser.nextToken() != XContentParser.Token.END_OBJECT) { diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java index 8fd1f44286094..b19d84815ded0 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -36,17 +36,27 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.cluster.Diff; +import org.opensearch.cluster.SnapshotDeletionsInProgress; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.cluster.SnapshotsInProgress.Entry; import org.opensearch.cluster.SnapshotsInProgress.ShardState; import org.opensearch.cluster.SnapshotsInProgress.State; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.repositories.IndexId; import org.opensearch.test.AbstractDiffableWireSerializationTestCase; import org.opensearch.test.VersionUtils; @@ -60,6 +70,9 @@ import java.util.Map; import java.util.stream.Collectors; +import static java.lang.Math.abs; +import static java.util.Collections.singletonMap; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_GATEWAY; import static org.opensearch.test.VersionUtils.randomVersion; public class SnapshotsInProgressSerializationTests extends AbstractDiffableWireSerializationTestCase { @@ -84,7 +97,7 @@ private Entry randomSnapshot() { for (int i = 0; i < numberOfIndices; i++) { indices.add(new IndexId(randomAlphaOfLength(10), randomAlphaOfLength(10))); } - long startTime = randomLong(); + long startTime = abs(randomLong()); long repositoryStateId = randomLong(); Map builder = new HashMap<>(); final List esIndices = indices.stream() @@ -183,6 +196,41 @@ protected Custom mutateInstance(Custom instance) { return SnapshotsInProgress.of(entries); } + public void testToXContent() throws IOException { + SnapshotsInProgress sip = SnapshotsInProgress.of(List.of(randomSnapshot(), randomSnapshot())); + boolean humanReadable = false; + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + sip, + mediaType, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY)), + humanReadable + ); + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + SnapshotsInProgress parsed = SnapshotsInProgress.fromXContent(parser); + assertEquals(sip, parsed); + } + } + + public void testToXContent_deletion() throws IOException { + SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(List.of(new SnapshotId("name1", "uuid1")), "repo", 10000000L, 10000L, SnapshotDeletionsInProgress.State.WAITING); + SnapshotDeletionsInProgress sdip = SnapshotDeletionsInProgress.of(List.of(entry)); + boolean humanReadable = false; + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + sdip, + mediaType, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY)), + humanReadable + ); + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + SnapshotDeletionsInProgress parsed = SnapshotDeletionsInProgress.fromXContent(parser); + assertEquals(sdip, parsed); + } + } + public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry( new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), @@ -191,7 +239,7 @@ public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { SnapshotsInProgressSerializationTests.randomState(Map.of()), Collections.emptyList(), Collections.emptyList(), - Math.abs(randomLong()), + abs(randomLong()), randomIntBetween(0, 1000), Map.of(), null, From 633b183de78fc749915a2630f596a81f9a60705c Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 4 Jun 2024 14:25:53 +0530 Subject: [PATCH 112/133] Fix snapshot de/ser Signed-off-by: Shivansh Arora --- .../cluster/SnapshotDeletionsInProgress.java | 13 +++- .../cluster/SnapshotsInProgress.java | 75 +++++++++++++++---- .../org/opensearch/repositories/IndexId.java | 2 +- ...SnapshotsInProgressSerializationTests.java | 52 ++++++++++++- 4 files changed, 122 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java index fed4332e99361..b59e0d8214932 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotDeletionsInProgress.java @@ -209,6 +209,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repository_state_id", entry.repositoryStateId); if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { builder.field("state", entry.state().value); + builder.field("uuid", entry.uuid()); } // else we don't serialize it } builder.endObject(); @@ -218,6 +219,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } ensureFieldName(parser, parser.currentToken(), TYPE); parser.nextToken(); ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); @@ -229,6 +236,7 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th byte stateValue = -1; List snapshotIds = new ArrayList<>(); TimeValue startTime = null; + String entryUUID = null; while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); final String fieldName = parser.currentName(); @@ -270,12 +278,15 @@ public static SnapshotDeletionsInProgress fromXContent(XContentParser parser) th case "state": stateValue = (byte) parser.intValue(); break; + case "uuid": + entryUUID = parser.text(); + break; default: throw new IllegalArgumentException("unknown field [" + fieldName + "]"); } } assert startTime != null; - entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue))); + entries.add(new Entry(snapshotIds, repository, startTime.millis(), repositoryStateId, State.fromValue(stateValue), entryUUID)); } return SnapshotDeletionsInProgress.of(entries); } diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java index 700b25d3d1e54..9a3f582995814 100644 --- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java @@ -42,6 +42,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -66,6 +67,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.common.xcontent.XContentUtils.readValue; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.core.xcontent.XContentParserUtils.ensureFieldName; import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; @@ -740,7 +742,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(STATE, status.state()); builder.field(NODE, status.nodeId()); if (params.param(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_API).equals(Metadata.CONTEXT_MODE_GATEWAY)) { - builder.field(INDEX_UUID, shardId.getIndex().getUUID()); if (status.generation() != null) builder.field(GENERATION, status.generation()); if (status.reason() != null) builder.field(REASON, status.reason()); } @@ -770,6 +771,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (status.reason() != null) builder.field(REASON, status.reason()); builder.endObject(); } + builder.endArray(); builder.field(REMOTE_STORE_INDEX_SHALLOW_COPY, remoteStoreIndexShallowCopy); } builder.endObject(); @@ -786,7 +788,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { Version version = null; SnapshotId source = null; Map metadata = null; - byte state = -1; + State state = null; List indices = new ArrayList<>(); long startTime = 0; long repositoryStateId = -1L; @@ -794,11 +796,10 @@ public static Entry fromXContent(XContentParser parser) throws IOException { List dataStreams = new ArrayList<>(); Map clones = new HashMap<>(); boolean remoteStoreIndexShallowCopy = false; - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { String currentFieldName = parser.currentName(); parser.nextToken(); - switch (currentFieldName) { case REPOSITORY: repository = parser.text(); @@ -816,7 +817,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { partial = parser.booleanValue(); break; case STATE: - state = (byte) parser.intValue(); + state = State.fromString(parser.text()); break; case INDICES: ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); @@ -836,32 +837,28 @@ public static Entry fromXContent(XContentParser parser) throws IOException { case SHARDS: ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - String index = null; - String indexUUID = null; + Index index = null; int shardId = -1; String nodeId = null; ShardState shardState = null; String reason = null; String generation = null; - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { final String currentShardField = parser.currentName(); parser.nextToken(); switch (currentShardField) { case INDEX: - index = parser.text(); + index = Index.fromXContent(parser); break; case SHARD: shardId = parser.intValue(); break; - case INDEX_UUID: - indexUUID = parser.text(); - break; case NODE: - nodeId = parser.text(); + nodeId = (String) readValue(parser, parser.currentToken()); break; case STATE: - shardState = ShardState.fromValue((byte) parser.intValue()); + shardState = ShardState.fromString(parser.text()); break; case REASON: reason = parser.text(); @@ -873,7 +870,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { throw new IllegalArgumentException("unknown field [" + currentShardField + "]"); } } - shards.put(new ShardId(index, indexUUID, shardId), + shards.put(new ShardId(index, shardId), reason != null ? new ShardSnapshotStatus(nodeId, shardState, reason, generation) : new ShardSnapshotStatus(nodeId, shardState, generation)); } @@ -950,7 +947,7 @@ public static Entry fromXContent(XContentParser parser) throws IOException { snapshot, includeGlobalState, partial, - State.fromValue(state), + state, indices, dataStreams, startTime, @@ -1203,6 +1200,25 @@ public static State fromValue(byte value) { throw new IllegalArgumentException("No snapshot state for value [" + value + "]"); } } + + public static State fromString(String value) { + switch(value) { + case "INIT": + return INIT; + case "STARTED": + return STARTED; + case "SUCCESS": + return SUCCESS; + case "FAILED": + return FAILED; + case "ABORTED": + return ABORTED; + case "PARTIAL": + return PARTIAL; + default: + throw new IllegalArgumentException("No snapshot state for value [" + value + "]"); + } + } } private final List entries; @@ -1311,6 +1327,12 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } public static SnapshotsInProgress fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { // fresh parser? move to the first token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } ensureFieldName(parser, parser.currentToken(), SNAPSHOTS); ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser); List entries = new ArrayList<>(); @@ -1380,5 +1402,26 @@ public static ShardState fromValue(byte value) { throw new IllegalArgumentException("No shard snapshot state for value [" + value + "]"); } } + + public static ShardState fromString(String state) { + switch (state) { + case "INIT": + return INIT; + case "SUCCESS": + return SUCCESS; + case "FAILED": + return FAILED; + case "ABORTED": + return ABORTED; + case "MISSING": + return MISSING; + case "WAITING": + return WAITING; + case "QUEUED": + return QUEUED; + default: + throw new IllegalArgumentException("No shard snapshot state for value [" + state + "]"); + } + } } } diff --git a/server/src/main/java/org/opensearch/repositories/IndexId.java b/server/src/main/java/org/opensearch/repositories/IndexId.java index 1a2e797c62847..2f97368d42038 100644 --- a/server/src/main/java/org/opensearch/repositories/IndexId.java +++ b/server/src/main/java/org/opensearch/repositories/IndexId.java @@ -138,7 +138,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par } public static IndexId fromXContent(XContentParser parser) throws IOException { - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); String name = null; String id = null; while (parser.nextToken() != XContentParser.Token.END_OBJECT) { diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java index 8fd1f44286094..b19d84815ded0 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotsInProgressSerializationTests.java @@ -36,17 +36,27 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.cluster.Diff; +import org.opensearch.cluster.SnapshotDeletionsInProgress; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.cluster.SnapshotsInProgress.Entry; import org.opensearch.cluster.SnapshotsInProgress.ShardState; import org.opensearch.cluster.SnapshotsInProgress.State; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.repositories.IndexId; import org.opensearch.test.AbstractDiffableWireSerializationTestCase; import org.opensearch.test.VersionUtils; @@ -60,6 +70,9 @@ import java.util.Map; import java.util.stream.Collectors; +import static java.lang.Math.abs; +import static java.util.Collections.singletonMap; +import static org.opensearch.cluster.metadata.Metadata.CONTEXT_MODE_GATEWAY; import static org.opensearch.test.VersionUtils.randomVersion; public class SnapshotsInProgressSerializationTests extends AbstractDiffableWireSerializationTestCase { @@ -84,7 +97,7 @@ private Entry randomSnapshot() { for (int i = 0; i < numberOfIndices; i++) { indices.add(new IndexId(randomAlphaOfLength(10), randomAlphaOfLength(10))); } - long startTime = randomLong(); + long startTime = abs(randomLong()); long repositoryStateId = randomLong(); Map builder = new HashMap<>(); final List esIndices = indices.stream() @@ -183,6 +196,41 @@ protected Custom mutateInstance(Custom instance) { return SnapshotsInProgress.of(entries); } + public void testToXContent() throws IOException { + SnapshotsInProgress sip = SnapshotsInProgress.of(List.of(randomSnapshot(), randomSnapshot())); + boolean humanReadable = false; + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + sip, + mediaType, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY)), + humanReadable + ); + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + SnapshotsInProgress parsed = SnapshotsInProgress.fromXContent(parser); + assertEquals(sip, parsed); + } + } + + public void testToXContent_deletion() throws IOException { + SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(List.of(new SnapshotId("name1", "uuid1")), "repo", 10000000L, 10000L, SnapshotDeletionsInProgress.State.WAITING); + SnapshotDeletionsInProgress sdip = SnapshotDeletionsInProgress.of(List.of(entry)); + boolean humanReadable = false; + XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint(); + final MediaType mediaType = MediaTypeRegistry.JSON; + BytesReference originalBytes = toShuffledXContent( + sdip, + mediaType, + new ToXContent.MapParams(singletonMap(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_GATEWAY)), + humanReadable + ); + try (XContentParser parser = createParser(mediaType.xContent(), originalBytes)) { + SnapshotDeletionsInProgress parsed = SnapshotDeletionsInProgress.fromXContent(parser); + assertEquals(sdip, parsed); + } + } + public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry( new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), @@ -191,7 +239,7 @@ public void testSerDeRemoteStoreIndexShallowCopy() throws IOException { SnapshotsInProgressSerializationTests.randomState(Map.of()), Collections.emptyList(), Collections.emptyList(), - Math.abs(randomLong()), + abs(randomLong()), randomIntBetween(0, 1000), Map.of(), null, From 1c0bbf911d01dc39891156229a592996e9388516 Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Tue, 4 Jun 2024 17:24:03 +0530 Subject: [PATCH 113/133] Add UTs for RemoteRoutingTableService Signed-off-by: Arpit Bandejiya --- .../routing/IndexShardRoutingTable.java | 2 +- .../remotestore/RemoteStoreNodeAttribute.java | 5 +- .../RemoteRoutingTableServiceTests.java | 104 +++++++++++++++++- 3 files changed, 104 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java index fd8cbea42c12f..2c0f8824ca4fc 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java @@ -738,7 +738,7 @@ public boolean equals(Object o) { IndexShardRoutingTable that = (IndexShardRoutingTable) o; if (!shardId.equals(that.shardId)) return false; - if (!shards.equals(that.shards)) return false; + if(!new HashSet<>(shards).equals(new HashSet<>(that.shards))) return false; return true; } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 2943667feeacf..bcd521bd8eef6 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -198,11 +198,14 @@ public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { } public static boolean isRemoteRoutingTableAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) .isEmpty() == false; } public static boolean isRemoteRoutingTableEnabled(Settings settings) { + assert FeatureFlags.isEnabled(REMOTE_ROUTING_TABLE_EXPERIMENTAL) == true; + assert RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) == true; + assert isRemoteRoutingTableAttributePresent(settings) == true; return FeatureFlags.isEnabled(REMOTE_ROUTING_TABLE_EXPERIMENTAL) && RemoteRoutingTableService.REMOTE_ROUTING_TABLE_ENABLED_SETTING.get(settings) && isRemoteRoutingTableAttributePresent(settings); } diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 162cf9b70cff8..3931fd5844c47 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -10,16 +10,30 @@ import org.junit.After; import org.junit.Before; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; import org.opensearch.repositories.FilterRepository; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; import static org.mockito.Mockito.*; @@ -33,6 +47,7 @@ public class RemoteRoutingTableServiceTests extends OpenSearchTestCase { private Supplier repositoriesServiceSupplier; private RepositoriesService repositoriesService; private BlobStoreRepository blobStoreRepository; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); @Before public void setup() { @@ -56,7 +71,7 @@ public void setup() { remoteRoutingTableService = new RemoteRoutingTableService( repositoriesServiceSupplier, settings, - new ThreadPool(settings) + threadPool ); } @@ -64,6 +79,7 @@ public void setup() { public void teardown() throws Exception { super.tearDown(); remoteRoutingTableService.close(); + threadPool.shutdown(); } @@ -79,10 +95,6 @@ public void testFailInitializationWhenRemoteRoutingDisabled() { ); } - public void testFailStartWhenRepositoryNotSet() { - doThrow(new RepositoryMissingException("repository missing")).when(repositoriesService).repository("routing_repository"); - assertThrows(RepositoryMissingException.class, () -> remoteRoutingTableService.start()); - } public void testFailStartWhenNotBlobRepository() { final FilterRepository filterRepository = mock(FilterRepository.class); @@ -90,4 +102,86 @@ public void testFailStartWhenNotBlobRepository() { assertThrows(AssertionError.class, () -> remoteRoutingTableService.start()); } + public void testGetChangedIndicesRouting() { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final Index index = new Index(indexName, "uuid"); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(1).numberOfReplicas(1).build(); + + RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).routingTable(routingTable).build(); + + assertEquals(0, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), state.getRoutingTable()).getUpserts().size()); + + //Reversing order to check for equality without order. + IndexRoutingTable indexRouting = routingTable.getIndicesRouting().get(indexName); + IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(index).addShard(indexRouting.getShards().get(0).replicaShards().get(0)) + .addShard(indexRouting.getShards().get(0).primaryShard()).build(); + ClusterState newState = ClusterState.builder(ClusterName.DEFAULT).routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build(); + assertEquals(0, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), newState.getRoutingTable()).getUpserts().size()); + } + + public void testIndicesRoutingDiffWhenIndexDeleted() { + + ClusterState state = createIndices(randomIntBetween(1,100)); + RoutingTable routingTable = state.routingTable(); + + List allIndices = new ArrayList<>(); + routingTable.getIndicesRouting().forEach((k,v) -> allIndices.add(k)); + + String indexNameToDelete = allIndices.get(randomIntBetween(0, allIndices.size()-1)); + RoutingTable updatedRoutingTable = RoutingTable.builder(routingTable).remove(indexNameToDelete).build(); + + assertEquals(1, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().size()); + assertEquals(indexNameToDelete, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().get(0)); + } + + public void testIndicesRoutingDiffWhenIndexDeletedAndAdded() { + + ClusterState state = createIndices(randomIntBetween(1,100)); + RoutingTable routingTable = state.routingTable(); + + List allIndices = new ArrayList<>(); + routingTable.getIndicesRouting().forEach((k,v) -> allIndices.add(k)); + + String indexNameToDelete = allIndices.get(randomIntBetween(0, allIndices.size()-1)); + RoutingTable.Builder updatedRoutingTableBuilder = RoutingTable.builder(routingTable).remove(indexNameToDelete); + + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(1).numberOfReplicas(1).build(); + + RoutingTable updatedRoutingTable = updatedRoutingTableBuilder.addAsNew(indexMetadata).build(); + + assertEquals(1, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().size()); + assertEquals(indexNameToDelete, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getDeletes().get(0)); + + assertEquals(1, RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getUpserts().size()); + assertTrue(RemoteRoutingTableService.getIndicesRoutingMapDiff(state.getRoutingTable(), updatedRoutingTable).getUpserts().containsKey(indexName)); + } + + private ClusterState createIndices(int numberOfIndices) { + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + for(int i=0; i< numberOfIndices; i++) { + String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); + final Index index = new Index(indexName, "uuid"); + final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") + .build() + ).numberOfShards(1).numberOfReplicas(1).build(); + + routingTableBuilder.addAsNew(indexMetadata); + } + return ClusterState.builder(ClusterName.DEFAULT).routingTable(routingTableBuilder.build()).build(); + } } From 0333312a6779c0be18b6e963808acf65569b07bc Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 5 Jun 2024 08:07:12 +0530 Subject: [PATCH 114/133] Make arraylist synchronized Signed-off-by: Sooraj Sinha --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c835fe83b3e06..620a1d46f2307 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -984,8 +984,8 @@ private ClusterState readClusterStateInParallel( readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0) + (readTransientSettingsMetadata ? 1 : 0); CountDownLatch latch = new CountDownLatch(totalReadTasks); List> asyncMetadataReadActions = new ArrayList<>(); - List readResults = new ArrayList<>(); - List readIndexRoutingTableResults = new ArrayList<>(); + List readResults = Collections.synchronizedList(new ArrayList<>()); + List readIndexRoutingTableResults = Collections.synchronizedList(new ArrayList<>()); List exceptionList = Collections.synchronizedList(new ArrayList<>(totalReadTasks)); LatchedActionListener listener = new LatchedActionListener<>( From 148eca559eed927a6fabd3833ccad553f84b6cd6 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 5 Jun 2024 08:22:35 +0530 Subject: [PATCH 115/133] Update remote publication to skip download on master node --- .../PublicationTransportHandler.java | 23 ++++++++++++++++--- .../coordination/RemotePublishRequest.java | 6 +++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index d553324bbf49d..bcad827517033 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -229,6 +229,9 @@ private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportReque } private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublishRequest request) throws IOException { + if (transportService.getLocalNode().equals(request.getSourceNode())) { + return acceptStateOnLocalNode(request); + } final Optional manifestOptional = remoteClusterStateService.getClusterMetadataManifestByTermVersion(request.getClusterName(), request.getClusterUUID(), request.term, request.version); if (manifestOptional.isPresent() == false) { throw new IllegalStateException( @@ -250,15 +253,17 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish } if (applyFullState == true) { + logger.info("Downloading full cluster state for term {}, version {}, stateUUID {}", manifest.getClusterTerm(), manifest.getStateVersion(), + manifest.getStateUUID()); ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest, transportService.getLocalNode().getId(), true); - logger.debug("Downloaded full cluster state [{}]", clusterState); fullClusterStateReceivedCount.incrementAndGet(); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.set(clusterState); return response; } else { - ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeenClusterState.get(), transportService.getLocalNode().getId()); - logger.debug("Downloaded full cluster state from diff [{}]", clusterState); + logger.info("Downloading diff cluster state for term {}, version {}, previousUUID {}, current UUID {}", manifest.getClusterTerm(), + manifest.getStateVersion(), manifest.getDiffManifest().getFromStateUUID(), manifest.getStateUUID()); + ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeen, transportService.getLocalNode().getId()); compatibleClusterStateDiffReceivedCount.incrementAndGet(); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.compareAndSet(lastSeen, clusterState); @@ -279,6 +284,18 @@ private PublishWithJoinResponse acceptState(ClusterState incomingState) { return handlePublishRequest.apply(new PublishRequest(incomingState)); } + private PublishWithJoinResponse acceptStateOnLocalNode(RemotePublishRequest remotePublishRequest) { + // if the state is coming from the current node, use original request instead (see currentPublishRequestToSelf for explanation) + final PublishRequest publishRequest = currentPublishRequestToSelf.get(); + if (publishRequest == null || publishRequest.getAcceptedState().coordinationMetadata().term() != remotePublishRequest.term + || publishRequest.getAcceptedState().version() != remotePublishRequest.version) { + throw new IllegalStateException("publication to self failed for " + remotePublishRequest); + } + PublishWithJoinResponse publishWithJoinResponse = handlePublishRequest.apply(publishRequest); + lastSeenClusterState.set(publishRequest.getAcceptedState()); + return publishWithJoinResponse; + } + public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemoteStateEnabled) { final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemoteStateEnabled); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java index b1981c0b4b1a0..e59f7cedd98e5 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java @@ -38,6 +38,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(clusterUUID); } + @Override + public String toString() { + return "RemotePublishRequest{" + "term=" + term + ", version=" + version + ", clusterName=" + clusterName + ", clusterUUID=" + clusterUUID + + ", sourceNode=" + sourceNode + '}'; + } + public String getClusterName() { return clusterName; } From 23f49e58cec2d9a977eb4718545521440ada7b64 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 5 Jun 2024 16:34:37 +0530 Subject: [PATCH 116/133] Add unit tests for uploadBlobAsync --- .../model/RemoteClusterStateBlobStore.java | 2 +- .../transfer/BlobStoreTransferService.java | 80 +++++----- .../translog/transfer/TransferService.java | 3 +- .../BlobStoreTransferServiceTests.java | 144 +++++++++++++++++- 4 files changed, 176 insertions(+), 53 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java index 62c706a18c1e8..567e522482a7d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java @@ -53,7 +53,7 @@ public void writeAsync(final U entity, final ActionListener listener) { try (InputStream inputStream = entity.serialize()) { BlobPath blobPath = getBlobPathForUpload(entity); entity.setFullBlobName(blobPath); - transferService.uploadBlob(inputStream, getBlobPathForUpload(entity), entity.getBlobFileName(), WritePriority.URGENT, listener); + transferService.uploadBlobAsync(inputStream, getBlobPathForUpload(entity), entity.getBlobFileName(), WritePriority.URGENT, listener); } } catch (Exception e) { listener.onFailure(e); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index 0babe8800fd16..318e204daba65 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -20,14 +20,12 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.blobstore.FetchBlobResult; -import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeFileInputStream; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.store.exception.ChecksumCombinationException; import org.opensearch.index.translog.ChannelFactory; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; @@ -111,7 +109,7 @@ public void uploadBlobs( } @Override - public void uploadBlob(InputStream inputStream, Iterable remotePath, String blobName, WritePriority writePriority, ActionListener listener) throws IOException { + public void uploadBlobAsync(InputStream inputStream, Iterable remotePath, String blobName, WritePriority writePriority, ActionListener listener) throws IOException { assert remotePath instanceof BlobPath; BlobPath blobPath = (BlobPath) remotePath; final BlobContainer blobContainer = blobStore.blobContainer(blobPath); @@ -135,20 +133,15 @@ public void uploadBlob(InputStream inputStream, Iterable remotePath, Str ); } - try ( - RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( - blobName, - blobName, - bytes.length, - true, - writePriority, - (size, position) -> new OffsetRangeIndexInputStream(input, size, position), - expectedChecksum, - ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported() - ) - ) { - ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), listener); - } + asyncBlobUpload(blobName, + blobName, + bytes.length, + blobPath, + writePriority, + (size, position) -> new OffsetRangeIndexInputStream(input, size, position), + expectedChecksum, + listener + ); } } @@ -165,36 +158,21 @@ private void uploadBlob( try (FileChannel channel = channelFactory.open(fileSnapshot.getPath(), StandardOpenOption.READ)) { contentLength = channel.size(); } - boolean remoteIntegrityEnabled = false; - BlobContainer blobContainer = blobStore.blobContainer(blobPath); - if (blobContainer instanceof AsyncMultiStreamBlobContainer) { - remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported(); - } - RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( - fileSnapshot.getName(), - fileSnapshot.getName(), - contentLength, - true, - writePriority, - (size, position) -> new OffsetRangeFileInputStream(fileSnapshot.getPath(), size, position), - Objects.requireNonNull(fileSnapshot.getChecksum()), - remoteIntegrityEnabled - ); ActionListener completionListener = ActionListener.wrap(resp -> listener.onResponse(fileSnapshot), ex -> { logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", fileSnapshot.getName()), ex); listener.onFailure(new FileTransferException(fileSnapshot, ex)); }); - completionListener = ActionListener.runBefore(completionListener, () -> { - try { - remoteTransferContainer.close(); - } catch (Exception e) { - logger.warn("Error occurred while closing streams", e); - } - }); - - WriteContext writeContext = remoteTransferContainer.createWriteContext(); - ((AsyncMultiStreamBlobContainer) blobStore.blobContainer(blobPath)).asyncBlobUpload(writeContext, completionListener); + Objects.requireNonNull(fileSnapshot.getChecksum()); + asyncBlobUpload(fileSnapshot.getName(), + fileSnapshot.getName(), + contentLength, + blobPath, + writePriority, + (size, position) -> new OffsetRangeFileInputStream(fileSnapshot.getPath(), size, position), + fileSnapshot.getChecksum(), + completionListener + ); } catch (Exception e) { logger.error(() -> new ParameterizedMessage("Failed to upload blob {}", fileSnapshot.getName()), e); @@ -209,6 +187,24 @@ private void uploadBlob( } + private void asyncBlobUpload(String fileName, String remoteFileName, long contentLength, BlobPath blobPath, WritePriority writePriority, RemoteTransferContainer.OffsetRangeInputStreamSupplier inputStreamSupplier, long expectedChecksum, ActionListener completionListener) throws IOException { + BlobContainer blobContainer = blobStore.blobContainer(blobPath); + assert blobContainer instanceof AsyncMultiStreamBlobContainer; + boolean remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported(); + try (RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( + fileName, + remoteFileName, + contentLength, + true, + writePriority, + inputStreamSupplier, + expectedChecksum, + remoteIntegrityEnabled + )) { + ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), completionListener); + } + } + @Override public InputStream downloadBlob(Iterable path, String fileName) throws IOException { return blobStore.blobContainer((BlobPath) path).readBlob(fileName); diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java index 9bed1cc822fda..c97690dfa8590 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java @@ -14,7 +14,6 @@ import org.opensearch.common.blobstore.FetchBlobResult; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; import java.io.IOException; @@ -66,7 +65,7 @@ void uploadBlobs( * @throws IOException the exception while transferring the data */ void uploadBlob(final TransferFileSnapshot fileSnapshot, Iterable remotePath, WritePriority writePriority) throws IOException; - void uploadBlob(InputStream inputStream, Iterable remotePath, String blobName, WritePriority writePriority, ActionListener listener) throws IOException; + void uploadBlobAsync(InputStream inputStream, Iterable remotePath, String blobName, WritePriority writePriority, ActionListener listener) throws IOException; void deleteBlobs(Iterable path, List fileNames) throws IOException; diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java index e4f5a454b15f6..ba10d423779cd 100644 --- a/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java +++ b/server/src/test/java/org/opensearch/index/translog/transfer/BlobStoreTransferServiceTests.java @@ -8,32 +8,52 @@ package org.opensearch.index.translog.transfer; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.opensearch.Version; import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.blobstore.fs.FsBlobContainer; +import org.opensearch.common.blobstore.fs.FsBlobStore; +import org.opensearch.common.blobstore.stream.read.ReadContext; +import org.opensearch.common.blobstore.stream.write.WriteContext; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.index.Index; import org.opensearch.env.Environment; import org.opensearch.env.TestEnvironment; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.repositories.Repository; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.blobstore.BlobStoreTestUtil; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - public class BlobStoreTransferServiceTests extends OpenSearchTestCase { private ThreadPool threadPool; @@ -103,6 +123,64 @@ public void onFailure(Exception e) { assertTrue(succeeded.get()); } + public void testUploadBlobFromInputStreamSyncFSRepo() throws IOException, InterruptedException{ + TransferService transferService = new BlobStoreTransferService(repository.blobStore(), threadPool); + uploadBlobFromInputStream(transferService); + } + + public void testUploadBlobFromInputStreamAsyncFSRepo() throws IOException, InterruptedException{ + BlobStore blobStore = createTestBlobStore(); + MockAsyncFsContainer mockAsyncFsContainer = new MockAsyncFsContainer( + (FsBlobStore) blobStore, + BlobPath.cleanPath(), + null + ); + FsBlobStore fsBlobStore = mock(FsBlobStore.class); + when(fsBlobStore.blobContainer(any())).thenReturn(mockAsyncFsContainer); + + TransferService transferService = new BlobStoreTransferService(fsBlobStore, threadPool); + uploadBlobFromInputStream(transferService); + } + + private IndexMetadata getIndexMetadata() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); + return new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .version(5L) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + } + + private void uploadBlobFromInputStream(TransferService transferService) throws IOException, InterruptedException { + TestClass testObject = new TestClass("field1", "value1"); + AtomicBoolean succeeded = new AtomicBoolean(false); + ChecksumBlobStoreFormat blobStoreFormat = new ChecksumBlobStoreFormat<>("coordination", "%s", IndexMetadata::fromXContent); + IndexMetadata indexMetadata = getIndexMetadata(); + try (InputStream inputStream = blobStoreFormat.serialize(indexMetadata, "index-metadata", new NoneCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput()) { + CountDownLatch latch = new CountDownLatch(1); + ActionListener listener = new LatchedActionListener<>(new ActionListener<>() { + @Override + public void onResponse(TestClass testObject) { + assert succeeded.compareAndSet(false, true); + assert testObject.name.equals("field1"); + } + + @Override + public void onFailure(Exception e) { + throw new AssertionError("Failed to perform uploadBlobAsync", e); + } + }, latch); + ActionListener completionListener = ActionListener.wrap(resp -> listener.onResponse(testObject), ex -> listener.onFailure(ex)); + transferService.uploadBlobAsync(inputStream, repository.basePath(), "test-object", WritePriority.URGENT, completionListener); + assertTrue(latch.await(1000, TimeUnit.MILLISECONDS)); + assertTrue(succeeded.get()); + } + } + @Override public void tearDown() throws Exception { super.tearDown(); @@ -134,6 +212,10 @@ protected void assertSnapshotOrGenericThread() { return repository; } + private BlobStore createTestBlobStore() throws IOException { + return new FsBlobStore(randomIntBetween(1, 8) * 1024, createTempDir(), false); + } + /** Create a {@link Environment} with random path.home and path.repo **/ private Environment createEnvironment() { Path home = createTempDir(); @@ -144,4 +226,50 @@ private Environment createEnvironment() { .build() ); } + + private static class TestClass implements Serializable { + private TestClass(String name, String value) { + this.name = name; + this.value = value; + } + + private final String name; + private final String value; + + @Override + public String toString() { + return "TestClass{ name: " + name + ", value: " + value + " }"; + } + } + + private static class MockAsyncFsContainer extends FsBlobContainer implements AsyncMultiStreamBlobContainer { + + private BlobContainer delegate; + + public MockAsyncFsContainer(FsBlobStore blobStore, BlobPath blobPath, Path path) { + super(blobStore, blobPath, path); + delegate = blobStore.blobContainer(BlobPath.cleanPath()); + } + + @Override + public void asyncBlobUpload(WriteContext writeContext, ActionListener completionListener) throws IOException { + InputStream inputStream = writeContext.getStreamProvider(Integer.MAX_VALUE).provideStream(0).getInputStream(); + delegate.writeBlob(writeContext.getFileName(), inputStream, writeContext.getFileSize(), true); + completionListener.onResponse(null); + } + + @Override + public void readBlobAsync(String blobName, ActionListener listener) { + throw new RuntimeException("read not supported"); + } + + @Override + public boolean remoteIntegrityCheckSupported() { + return false; + } + + public BlobContainer getDelegate() { + return delegate; + } + } } From 7f2391c6a58831f7e7ed3cbfc4f5e0e5d4421d40 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 5 Jun 2024 17:47:38 +0530 Subject: [PATCH 117/133] Address interface comments --- .../AbstractRemoteWritableBlobEntity.java | 30 +++++++++---- .../remote/RemoteWritableEntityStore.java | 3 +- .../remote/RemoteClusterStateUtils.java | 3 ++ .../model/RemoteClusterStateBlobStore.java | 44 +++++++++++-------- 4 files changed, 52 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java index 2ef37a6af35c2..632b2b70d61df 100644 --- a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java +++ b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java @@ -8,13 +8,13 @@ package org.opensearch.common.remote; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; - import org.opensearch.common.blobstore.BlobPath; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; + /** * An extension of {@link RemoteWriteableEntity} class which caters to the use case of writing to and reading from a blob storage * @@ -28,8 +28,13 @@ public abstract class AbstractRemoteWritableBlobEntity implements RemoteWrite private final String clusterUUID; private final Compressor compressor; private final NamedXContentRegistry namedXContentRegistry; + private String[] pathTokens; - public AbstractRemoteWritableBlobEntity(final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public AbstractRemoteWritableBlobEntity( + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { this.clusterUUID = clusterUUID; this.compressor = compressor; this.namedXContentRegistry = namedXContentRegistry; @@ -43,11 +48,8 @@ public String getFullBlobName() { public String getBlobFileName() { if (blobFileName == null) { - if (blobName == null) { - return null; - } - String[] pathTokens = blobName.split(PATH_DELIMITER); - if (pathTokens.length < 1) { + String[] pathTokens = getBlobPathTokens(); + if (pathTokens == null || pathTokens.length < 1) { return null; } blobFileName = pathTokens[pathTokens.length - 1]; @@ -55,6 +57,17 @@ public String getBlobFileName() { return blobFileName; } + public String[] getBlobPathTokens() { + if (pathTokens != null) { + return pathTokens; + } + if (blobName == null) { + return null; + } + pathTokens = blobName.split(PATH_DELIMITER); + return pathTokens; + } + public abstract String generateBlobFileName(); public String clusterUUID() { @@ -70,6 +83,7 @@ public void setFullBlobName(BlobPath blobPath) { public NamedXContentRegistry getNamedXContentRegistry() { return namedXContentRegistry; } + protected Compressor getCompressor() { return compressor; } diff --git a/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java b/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java index 48db26292f6ae..78d6dd2d7b474 100644 --- a/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java +++ b/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java @@ -8,9 +8,10 @@ package org.opensearch.common.remote; -import java.io.IOException; import org.opensearch.core.action.ActionListener; +import java.io.IOException; + /** * An interface to read/write an object from/to a remote storage. This interface is agnostic of the remote storage type. * diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 94a57e6b0be77..67280c15f1040 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -22,6 +22,9 @@ import java.util.List; import java.util.Map; +/** + * Utility class for Remote Cluster State + */ public class RemoteClusterStateUtils { public static final String METADATA_NAME_FORMAT = "%s.dat"; public static final String METADATA_NAME_PLAIN_FORMAT = "%s"; diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java index 567e522482a7d..b391d20e21087 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java @@ -8,12 +8,6 @@ package org.opensearch.gateway.remote.model; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.PATH_DELIMITER; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.concurrent.ExecutorService; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; @@ -25,6 +19,10 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.ExecutorService; + /** * Abstract class for a blob type storage * @@ -38,8 +36,13 @@ public class RemoteClusterStateBlobStore listener) { @Override public U read(final U entity) throws IOException { - assert entity.getFullBlobName() != null; - T object = entity.deserialize( - transferService.downloadBlob(getBlobPathForDownload(entity), entity.getBlobFileName())); + // TODO Add timing logs and tracing + assert entity.get() == null && entity.getFullBlobName() != null; + T object = entity.deserialize(transferService.downloadBlob(getBlobPathForDownload(entity), entity.getBlobFileName())); entity.set(object); return entity; } @@ -81,7 +84,10 @@ public void readAsync(final U entity, final ActionListener listener) { } private BlobPath getBlobPathForUpload(final AbstractRemoteWritableBlobEntity obj) { - BlobPath blobPath = blobStoreRepository.basePath().add(RemoteClusterStateUtils.encodeString(clusterName)).add("cluster-state").add(obj.clusterUUID()); + BlobPath blobPath = blobStoreRepository.basePath() + .add(RemoteClusterStateUtils.encodeString(clusterName)) + .add("cluster-state") + .add(obj.clusterUUID()); for (String token : obj.getBlobPathParameters().getPathTokens()) { blobPath = blobPath.add(token); } @@ -89,16 +95,16 @@ private BlobPath getBlobPathForUpload(final AbstractRemoteWritableBlobEntity } private BlobPath getBlobPathForDownload(final AbstractRemoteWritableBlobEntity obj) { - String[] pathTokens = extractBlobPathTokens(obj.getFullBlobName()); + String[] pathTokens = obj.getBlobPathTokens(); BlobPath blobPath = new BlobPath(); - for (String token : pathTokens) { - blobPath = blobPath.add(token); + if (pathTokens == null || pathTokens.length < 1) { + return blobPath; + } + // Iterate till second last path token to get the blob folder + for (int i = 0; i < pathTokens.length - 1; i++) { + blobPath = blobPath.add(pathTokens[i]); } return blobPath; } - private static String[] extractBlobPathTokens(final String blobName) { - String[] blobNameTokens = blobName.split(PATH_DELIMITER); - return Arrays.copyOfRange(blobNameTokens, 0, blobNameTokens.length - 1); - } } From caf61a615e7f93602edfccb2b8ce5edb6b36acfb Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Tue, 4 Jun 2024 19:07:09 +0530 Subject: [PATCH 118/133] read cluster state custom from diff Signed-off-by: Shivansh Arora --- .../remote/ClusterStateDiffManifest.java | 12 ++++ .../RemoteClusterStateAttributesManager.java | 6 +- .../remote/RemoteClusterStateService.java | 63 ++++++++++++++++--- .../gateway/remote/RemoteManifestManager.java | 4 +- 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 35263adc540e2..ee883b56ae40b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -459,6 +459,10 @@ public boolean isDiscoveryNodesUpdated() { return discoveryNodesUpdated; } + public boolean isHashesOfConsistentSettingsUpdated() { + return hashesOfConsistentSettingsUpdated; + } + public List getIndicesRoutingUpdated() { return indicesRoutingUpdated; } @@ -467,6 +471,14 @@ public List getIndicesRoutingDeleted() { return indicesRoutingDeleted; } + public List getClusterStateCustomUpdated() { + return clusterStateCustomUpdated; + } + + public List getClusterStateCustomDeleted() { + return clusterStateCustomDeleted; + } + public static Builder builder() { return new Builder(); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index c02b5b3116f7a..4a4b0c79b21a9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -33,6 +33,7 @@ import java.util.Set; import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; public class RemoteClusterStateAttributesManager { public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; @@ -100,7 +101,7 @@ public CheckedRunnable getAsyncMetadataReadAction( String uploadedFilename, LatchedActionListener listener ) { - ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); + final ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); if (component.equals(RemoteDiscoveryNodes.DISCOVERY_NODES)) { RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); return () -> discoveryNodesBlobStore.readAsync(remoteDiscoveryNodes, actionListener); @@ -108,8 +109,9 @@ public CheckedRunnable getAsyncMetadataReadAction( RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); } else if (component.equals(CLUSTER_STATE_CUSTOM)) { + final ActionListener customActionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, String.join(CUSTOM_DELIMITER, component, componentName))), listener::onFailure); RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(uploadedFilename, componentName, clusterUUID, compressor, namedXContentRegistry); - return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, actionListener); + return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, customActionListener); } else { throw new RemoteStateTransferException("Remote object not found for "+ component); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 620a1d46f2307..4b5bcdeb7b6cf 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -637,12 +637,11 @@ private UploadedMetadataResults writeMetadataInParallel( ); }); clusterStateCustomToUpload.forEach((key, value) -> { - String customComponent = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, key); uploadTasks.put( - customComponent, + key, remoteClusterStateAttributesManager.getAsyncMetadataWriteAction( clusterState, - customComponent, + key, value, listener ) @@ -977,11 +976,14 @@ private ClusterState readClusterStateInParallel( boolean readTemplatesMetadata, boolean readDiscoveryNodes, boolean readClusterBlocks, - List indicesRoutingToRead + List indicesRoutingToRead, + boolean readHashesOfConsistentSettings, + Map clusterStateCustomToRead ) throws IOException { int totalReadTasks = indicesToRead.size() + customToRead.size() + indicesRoutingToRead.size() + (readCoordinationMetadata ? 1 : 0) + (readSettingsMetadata ? 1 : 0) + ( - readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0) + (readTransientSettingsMetadata ? 1 : 0); + readTemplatesMetadata ? 1 : 0) + (readDiscoveryNodes ? 1 : 0) + (readClusterBlocks ? 1 : 0) + (readTransientSettingsMetadata ? 1 : 0) + (readHashesOfConsistentSettings ? 1 : 0) + + clusterStateCustomToRead.size(); CountDownLatch latch = new CountDownLatch(totalReadTasks); List> asyncMetadataReadActions = new ArrayList<>(); List readResults = Collections.synchronizedList(new ArrayList<>()); @@ -1120,6 +1122,30 @@ private ClusterState readClusterStateInParallel( ); } + if (readHashesOfConsistentSettings) { + asyncMetadataReadActions.add( + remoteGlobalMetadataManager.getAsyncMetadataReadAction( + clusterUUID, + HASHES_OF_CONSISTENT_SETTINGS, + HASHES_OF_CONSISTENT_SETTINGS, + manifest.getHashesOfConsistentSettings().getUploadedFilename(), + listener + ) + ); + } + + for (Map.Entry entry : clusterStateCustomToRead.entrySet()) { + asyncMetadataReadActions.add( + remoteClusterStateAttributesManager.getAsyncMetadataReadAction( + clusterUUID, + CLUSTER_STATE_CUSTOM, + entry.getKey(), + entry.getValue().getUploadedFilename(), + listener + ) + ); + } + for (CheckedRunnable asyncMetadataReadAction : asyncMetadataReadActions) { asyncMetadataReadAction.run(); } @@ -1175,11 +1201,17 @@ private ClusterState readClusterStateInParallel( case TEMPLATES_METADATA: metadataBuilder.templates((TemplatesMetadata) remoteReadResult.getObj()); break; + case HASHES_OF_CONSISTENT_SETTINGS: + metadataBuilder.hashesOfConsistentSettings((DiffableStringMap) remoteReadResult.getObj()); case CLUSTER_STATE_ATTRIBUTE: if (remoteReadResult.getComponentName().equals(DISCOVERY_NODES)) { discoveryNodesBuilder.set(DiscoveryNodes.builder((DiscoveryNodes) remoteReadResult.getObj())); } else if (remoteReadResult.getComponentName().equals(CLUSTER_BLOCKS)) { clusterStateBuilder.blocks((ClusterBlocks) remoteReadResult.getObj()); + } else if (remoteReadResult.getComponentName().startsWith(CLUSTER_STATE_CUSTOM)) { + // component name for mat is "cluster-state-custom--custom_name" + String custom = remoteReadResult.getComponentName().split(CUSTOM_DELIMITER)[1]; + clusterStateBuilder.putCustom(custom, (ClusterState.Custom) remoteReadResult.getObj()); } break; default: @@ -1218,7 +1250,9 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada manifest.getTemplatesMetadata() != null, includeEphemeral && manifest.getDiscoveryNodesMetadata() != null, includeEphemeral && manifest.getClusterBlocksMetadata() != null, - includeEphemeral ? manifest.getIndicesRouting() : Collections.emptyList() + includeEphemeral ? manifest.getIndicesRouting() : Collections.emptyList(), + includeEphemeral && manifest.getHashesOfConsistentSettings() != null, + includeEphemeral ? manifest.getClusterStateCustomMap() : Collections.emptyMap() ); } @@ -1242,6 +1276,12 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata updatedCustomMetadata.put(customType, manifest.getCustomMetadataMap().get(customType)); } } + Map updatedClusterStateCustom = new HashMap<>(); + if (diff.getClusterStateCustomUpdated() != null) { + for (String customType : diff.getClusterStateCustomUpdated()) { + updatedClusterStateCustom.put(customType, manifest.getClusterStateCustomMap().get(customType)); + } + } ClusterState updatedClusterState = readClusterStateInParallel( previousState, manifest, @@ -1256,7 +1296,9 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata diff.isTemplatesMetadataUpdated(), diff.isDiscoveryNodesUpdated(), diff.isClusterBlocksUpdated(), - updatedIndexRouting + updatedIndexRouting, + diff.isHashesOfConsistentSettingsUpdated(), + updatedClusterStateCustom ); ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); Metadata.Builder metadataBuilder = Metadata.builder(updatedClusterState.metadata()); @@ -1271,6 +1313,13 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata } } + // remove the deleted cluster state customs from the metadata + if (diff.getClusterStateCustomDeleted() != null) { + for (String customType : diff.getClusterStateCustomDeleted()) { + clusterStateBuilder.removeCustom(customType); + } + } + HashMap indexRoutingTables = new HashMap<>(updatedClusterState.getRoutingTable().getIndicesRouting()); for (String indexName : diff.getIndicesRoutingDeleted()) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 8126c6f774f8a..3446f52472bdd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -101,7 +101,9 @@ ClusterMetadataManifest uploadManifest( .routingTableVersion(clusterState.getRoutingTable().version()) .indicesRouting(uploadedMetadataResult.uploadedIndicesRoutingMetadata) .metadataVersion(clusterState.metadata().version()) - .transientSettingsMetadata(uploadedMetadataResult.uploadedTransientSettingsMetadata); + .transientSettingsMetadata(uploadedMetadataResult.uploadedTransientSettingsMetadata) + .clusterStateCustomMetadataMap(uploadedMetadataResult.uploadedClusterStateCustomMetadataMap) + .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings); final ClusterMetadataManifest manifest = manifestBuilder.build(); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest); return manifest; From e1eddbe2d93c37633b6e67958759721e065466f1 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Wed, 5 Jun 2024 23:47:44 +0530 Subject: [PATCH 119/133] Remove get/set method from interface Signed-off-by: Sooraj Sinha --- .../common/remote/RemoteWritableEntityStore.java | 2 +- .../common/remote/RemoteWriteableEntity.java | 11 ----------- .../gateway/remote/RemoteGlobalMetadataManager.java | 10 +++++----- .../gateway/remote/RemoteIndexMetadataManager.java | 2 +- .../gateway/remote/RemoteManifestManager.java | 2 +- .../gateway/remote/model/RemoteClusterBlocks.java | 11 ----------- .../remote/model/RemoteClusterMetadataManifest.java | 10 ---------- .../remote/model/RemoteClusterStateBlobStore.java | 11 ++++------- .../remote/model/RemoteClusterStateCustoms.java | 10 ---------- .../remote/model/RemoteCoordinationMetadata.java | 10 ---------- .../gateway/remote/model/RemoteCustomMetadata.java | 10 ---------- .../gateway/remote/model/RemoteDiscoveryNodes.java | 10 ---------- .../gateway/remote/model/RemoteGlobalMetadata.java | 10 ---------- .../model/RemoteHashesOfConsistentSettings.java | 10 ---------- .../gateway/remote/model/RemoteIndexMetadata.java | 12 ------------ .../model/RemotePersistentSettingsMetadata.java | 10 ---------- .../remote/model/RemoteTemplatesMetadata.java | 10 ---------- .../model/RemoteTransientSettingsMetadata.java | 10 ---------- 18 files changed, 12 insertions(+), 149 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java b/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java index 78d6dd2d7b474..ccf7cafff1730 100644 --- a/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java +++ b/server/src/main/java/org/opensearch/common/remote/RemoteWritableEntityStore.java @@ -22,7 +22,7 @@ public interface RemoteWritableEntityStore public void writeAsync(U entity, ActionListener listener); - public U read(U entity) throws IOException; + public T read(U entity) throws IOException; public void readAsync(U entity, ActionListener listener); } diff --git a/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntity.java b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntity.java index 06818489a1f32..778c24dce2e27 100644 --- a/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntity.java +++ b/server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntity.java @@ -18,17 +18,6 @@ * @param The object type which can be uploaded to or downloaded from remote storage. */ public interface RemoteWriteableEntity { - - /** - * @param object The object T which is to be set in this writable entity - */ - public void set(T object); - - /** - * @return The entity T contained within this class - */ - public T get(); - /** * @return An InputStream created by serializing the entity T * @throws IOException Exception encountered while serialization diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index fb76bd3dc4f66..d1153a0a90809 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -193,7 +193,7 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe if (globalMetadataFileName != null) { RemoteGlobalMetadata remoteGlobalMetadata = new RemoteGlobalMetadata(globalMetadataFileName, clusterUUID, compressor, namedXContentRegistry); - return globalMetadataBlobStore.read(remoteGlobalMetadata).get(); + return globalMetadataBlobStore.read(remoteGlobalMetadata); } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { CoordinationMetadata coordinationMetadata = getCoordinationMetadata( clusterUUID, @@ -239,7 +239,7 @@ public CoordinationMetadata getCoordinationMetadata(String clusterUUID, String c if (coordinationMetadataFileName != null) { RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(coordinationMetadataFileName, clusterUUID, compressor, namedXContentRegistry); - return coordinationMetadataBlobStore.read(remoteCoordinationMetadata).get(); + return coordinationMetadataBlobStore.read(remoteCoordinationMetadata); } else { return CoordinationMetadata.EMPTY_METADATA; } @@ -257,7 +257,7 @@ public Settings getSettingsMetadata(String clusterUUID, String settingsMetadataF if (settingsMetadataFileName != null) { RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(settingsMetadataFileName, clusterUUID, compressor, namedXContentRegistry); - return persistentSettingsBlobStore.read(remotePersistentSettingsMetadata).get(); + return persistentSettingsBlobStore.read(remotePersistentSettingsMetadata); } else { return Settings.EMPTY; } @@ -275,7 +275,7 @@ public TemplatesMetadata getTemplatesMetadata(String clusterUUID, String templat if (templatesMetadataFileName != null) { RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(templatesMetadataFileName, clusterUUID, compressor, namedXContentRegistry); - return templatesMetadataBlobStore.read(remoteTemplatesMetadata).get(); + return templatesMetadataBlobStore.read(remoteTemplatesMetadata); } else { return TemplatesMetadata.EMPTY_METADATA; } @@ -292,7 +292,7 @@ public Metadata.Custom getCustomsMetadata(String clusterUUID, String customMetad try { // Fetch Custom metadata RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, compressor, namedXContentRegistry); - return customMetadataBlobStore.read(remoteCustomMetadata).get(); + return customMetadataBlobStore.read(remoteCustomMetadata); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java index 77d6c0d200c24..5a18c075acc7e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java @@ -93,7 +93,7 @@ IndexMetadata getIndexMetadata( RemoteIndexMetadata remoteIndexMetadata = new RemoteIndexMetadata(RemoteClusterStateUtils.getFormattedFileName( uploadedIndexMetadata.getUploadedFilename(), manifestCodecVersion), clusterUUID, compressor, namedXContentRegistry); try { - return indexMetadataBlobStore.read(remoteIndexMetadata).get(); + return indexMetadataBlobStore.read(remoteIndexMetadata); } catch (IOException e) { throw new IllegalStateException( String.format(Locale.ROOT, "Error while downloading IndexMetadata - %s", uploadedIndexMetadata.getUploadedFilename()), diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 3446f52472bdd..e6cbdcb5c31de 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -186,7 +186,7 @@ ClusterMetadataManifest fetchRemoteClusterMetadataManifest(String clusterName, S try { String fullBlobName = getManifestFolderPath(clusterName, clusterUUID).buildAsString() + filename; RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(fullBlobName, clusterUUID, compressor, namedXContentRegistry); - return manifestBlobStore.read(remoteClusterMetadataManifest).get(); + return manifestBlobStore.read(remoteClusterMetadataManifest); } catch (IOException e) { throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index f608fd0e352cc..c2fff1ee15439 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -79,17 +79,6 @@ public UploadedMetadata getUploadedMetadata() { return new UploadedMetadataAttribute(CLUSTER_BLOCKS, blobName); } - @Override - public void set(final ClusterBlocks clusterBlocks) { - this.clusterBlocks = clusterBlocks; - } - - @Override - public ClusterBlocks get() { - return clusterBlocks; - } - - @Override public InputStream serialize() throws IOException { return CLUSTER_BLOCKS_FORMAT.serialize(clusterBlocks, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index f55881ec8dc88..9343dea21fbe0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -100,16 +100,6 @@ public UploadedMetadata getUploadedMetadata() { return new UploadedMetadataAttribute(MANIFEST_PATH_TOKEN, blobName); } - @Override - public void set(final ClusterMetadataManifest manifest) { - this.clusterMetadataManifest = manifest; - } - - @Override - public ClusterMetadataManifest get() { - return clusterMetadataManifest; - } - @Override public InputStream serialize() throws IOException { return CLUSTER_METADATA_MANIFEST_FORMAT.serialize(clusterMetadataManifest, generateBlobFileName(), getCompressor(), diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java index b391d20e21087..aadd4f353853c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateBlobStore.java @@ -51,7 +51,6 @@ public RemoteClusterStateBlobStore( @Override public void writeAsync(final U entity, final ActionListener listener) { - assert entity.get() != null; try { try (InputStream inputStream = entity.serialize()) { BlobPath blobPath = getBlobPathForUpload(entity); @@ -64,19 +63,17 @@ public void writeAsync(final U entity, final ActionListener listener) { } @Override - public U read(final U entity) throws IOException { + public T read(final U entity) throws IOException { // TODO Add timing logs and tracing - assert entity.get() == null && entity.getFullBlobName() != null; - T object = entity.deserialize(transferService.downloadBlob(getBlobPathForDownload(entity), entity.getBlobFileName())); - entity.set(object); - return entity; + assert entity.getFullBlobName() != null; + return entity.deserialize(transferService.downloadBlob(getBlobPathForDownload(entity), entity.getBlobFileName())); } @Override public void readAsync(final U entity, final ActionListener listener) { executorService.execute(() -> { try { - listener.onResponse(read(entity).get()); + listener.onResponse(read(entity)); } catch (Exception e) { listener.onFailure(e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java index fffbcfab4e141..58fbbdad3c4b5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -86,16 +86,6 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { return new ClusterMetadataManifest.UploadedMetadataAttribute(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType), blobName); } - @Override - public void set(Custom custom) { - this.custom = custom; - } - - @Override - public ClusterState.Custom get() { - return custom; - } - @Override public InputStream serialize() throws IOException { return clusterStateCustomBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java index 6b7368457fe89..bdc8e1cf8ec30 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadata.java @@ -72,16 +72,6 @@ public String generateBlobFileName() { return blobFileName; } - @Override - public void set(final CoordinationMetadata coordinationMetadata) { - this.coordinationMetadata = coordinationMetadata; - } - - @Override - public CoordinationMetadata get() { - return coordinationMetadata; - } - @Override public InputStream serialize() throws IOException { return COORDINATION_METADATA_FORMAT.serialize(coordinationMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java index e0d84955b5ea4..7c1a3beb35755 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteCustomMetadata.java @@ -94,16 +94,6 @@ public String generateBlobFileName() { return blobFileName; } - @Override - public void set(final Custom custom) { - this.custom = custom; - } - - @Override - public Custom get() { - return custom; - } - @Override public InputStream serialize() throws IOException { return customBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index 34aca7767a0b1..e63e6ed63f6d1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -79,16 +79,6 @@ public UploadedMetadata getUploadedMetadata() { return new UploadedMetadataAttribute(DISCOVERY_NODES, blobName); } - @Override - public void set(final DiscoveryNodes discoveryNodes) { - this.discoveryNodes = discoveryNodes; - } - - @Override - public DiscoveryNodes get() { - return discoveryNodes; - } - @Override public InputStream serialize() throws IOException { return DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java index 42b8998bd6d32..9149a542383c1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadata.java @@ -53,16 +53,6 @@ public UploadedMetadata getUploadedMetadata() { throw new UnsupportedOperationException(); } - @Override - public void set(final Metadata metadata) { - this.metadata = metadata; - } - - @Override - public Metadata get() { - return metadata; - } - @Override public InputStream serialize() throws IOException { throw new UnsupportedOperationException(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java index 3dfa42139bb02..d7908ac6e8235 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -72,16 +72,6 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { return new ClusterMetadataManifest.UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, blobName); } - @Override - public void set(final DiffableStringMap hashesOfConsistentSettings) { - this.hashesOfConsistentSettings = hashesOfConsistentSettings; - } - - @Override - public DiffableStringMap get() { - return hashesOfConsistentSettings; - } - @Override public InputStream serialize() throws IOException { return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.serialize(hashesOfConsistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java index 6f42488b79fc0..d77831aaaea6f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java @@ -52,18 +52,6 @@ public RemoteIndexMetadata(final String blobName, final String clusterUUID, fina this.blobName = blobName; } - @Override - public void set(final IndexMetadata indexMetadata) { - this.indexMetadata = indexMetadata; - } - - @Override - public IndexMetadata get() { - return indexMetadata; - } - - - @Override public BlobPathParameters getBlobPathParameters() { return new BlobPathParameters(List.of(INDEX_PATH_TOKEN, indexMetadata.getIndexUUID()), "metadata"); diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java index b2a8f07087cc5..dd96afc457fa8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadata.java @@ -72,16 +72,6 @@ public String generateBlobFileName() { return blobFileName; } - @Override - public void set(final Settings settings) { - this.persistentSettings = settings; - } - - @Override - public Settings get() { - return persistentSettings; - } - @Override public InputStream serialize() throws IOException { return SETTINGS_METADATA_FORMAT.serialize(persistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadata.java index 6023600d33a4e..c558e822191b1 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadata.java @@ -73,16 +73,6 @@ public String generateBlobFileName() { return blobFileName; } - @Override - public void set(final TemplatesMetadata templatesMetadata) { - this.templatesMetadata = templatesMetadata; - } - - @Override - public TemplatesMetadata get() { - return templatesMetadata; - } - @Override public InputStream serialize() throws IOException { return TEMPLATES_METADATA_FORMAT.serialize(templatesMetadata, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java index 75b6cfc3a765a..957a1b3977aa4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java @@ -72,16 +72,6 @@ public String generateBlobFileName() { return blobFileName; } - @Override - public void set(final Settings settings) { - this.transientSettings = settings; - } - - @Override - public Settings get() { - return transientSettings; - } - @Override public InputStream serialize() throws IOException { return SETTINGS_METADATA_FORMAT.serialize(transientSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) From 8b71d2c8c886a045d446cf7b5c08979300f2af7c Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 6 Jun 2024 13:30:49 +0530 Subject: [PATCH 120/133] disable checksum for debugging --- .../remote/RemoteRoutingTableService.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 44c32f7830ae4..b2a2eb4401bc1 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -171,12 +171,13 @@ public CheckedRunnable getIndexRoutingAsyncAction( try ( InputStream indexRoutingStream = new IndexRoutingTableInputStream(indexRouting); IndexInput input = new ByteArrayIndexInput("indexrouting", indexRoutingStream.readAllBytes())) { - long expectedChecksum; - try { - expectedChecksum = checksumOfChecksum(input.clone(), 8); - } catch (Exception e) { - throw e; - } + //todo reenable checksum +// long expectedChecksum; +// try { +// expectedChecksum = checksumOfChecksum(input.clone(), 8); +// } catch (Exception e) { +// throw e; +// } try ( RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer( fileName, @@ -185,8 +186,10 @@ public CheckedRunnable getIndexRoutingAsyncAction( true, WritePriority.URGENT, (size, position) -> new OffsetRangeIndexInputStream(input, size, position), - expectedChecksum, - ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported() +// expectedChecksum, +// ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported() + null, + false ) ) { return () -> ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), completionListener); From 87ade4cdaa09d96260a56d2f45d78cd56c1817d3 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Thu, 6 Jun 2024 13:38:13 +0530 Subject: [PATCH 121/133] Use byte buffer --- .../transfer/BlobStoreTransferService.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java index 71a0556910810..86f16b6d6d04a 100644 --- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java +++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java @@ -9,12 +9,16 @@ package org.opensearch.index.translog.transfer; import java.io.ByteArrayOutputStream; +import java.util.Arrays; import java.util.Base64; import java.util.HashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.ByteBuffersDataInput; +import org.apache.lucene.store.ByteBuffersIndexInput; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BytesRef; import org.opensearch.action.ActionRunnable; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer; @@ -27,8 +31,10 @@ import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeFileInputStream; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; +import org.opensearch.common.io.Streams; import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.store.exception.ChecksumCombinationException; import org.opensearch.index.translog.ChannelFactory; import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot; @@ -113,18 +119,21 @@ public void uploadBlobs( } @Override - public void uploadBlobAsync(InputStream inputStream, Iterable remotePath, String blobName, WritePriority writePriority, ActionListener listener) throws IOException { + public void uploadBlobAsync(InputStream inputStream, Iterable remotePath, String fileName, WritePriority writePriority, + ActionListener listener) throws IOException { assert remotePath instanceof BlobPath; BlobPath blobPath = (BlobPath) remotePath; final BlobContainer blobContainer = blobStore.blobContainer(blobPath); if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) { - blobContainer.writeBlob(blobName, inputStream, inputStream.available(), false); + blobContainer.writeBlob(fileName, inputStream, inputStream.available(), false); listener.onResponse(null); return; } - final String resourceDescription = "BlobStoreTransferService.uploadBlob(blob=\"" + blobName + "\")"; - byte[] bytes = inputStream.readAllBytes(); - try (IndexInput input = new ByteArrayIndexInput(resourceDescription, bytes)) { + final String resourceDescription = "BlobStoreTransferService.uploadBlob(blob=\"" + fileName + "\")"; + try (IndexInput input = inputStream.available() > 0 + ? new ByteBuffersIndexInput(new ByteBuffersDataInput(Arrays.asList(BytesReference.toByteBuffers(Streams.readFully(inputStream)))), + resourceDescription) + : new ByteArrayIndexInput(resourceDescription, BytesRef.EMPTY_BYTES)) { long expectedChecksum; try { expectedChecksum = checksumOfChecksum(input.clone(), 8); @@ -137,9 +146,9 @@ public void uploadBlobAsync(InputStream inputStream, Iterable remotePath ); } - asyncBlobUpload(blobName, - blobName, - bytes.length, + asyncBlobUpload(fileName, + fileName, + inputStream.available(), blobPath, writePriority, (size, position) -> new OffsetRangeIndexInputStream(input, size, position), @@ -222,7 +231,9 @@ private void uploadBlob( } - private void asyncBlobUpload(String fileName, String remoteFileName, long contentLength, BlobPath blobPath, WritePriority writePriority, RemoteTransferContainer.OffsetRangeInputStreamSupplier inputStreamSupplier, long expectedChecksum, ActionListener completionListener, Map metadata) throws IOException { + private void asyncBlobUpload(String fileName, String remoteFileName, long contentLength, BlobPath blobPath, WritePriority writePriority, + RemoteTransferContainer.OffsetRangeInputStreamSupplier inputStreamSupplier, Long expectedChecksum, ActionListener completionListener, + Map metadata) throws IOException { BlobContainer blobContainer = blobStore.blobContainer(blobPath); assert blobContainer instanceof AsyncMultiStreamBlobContainer; boolean remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) blobContainer).remoteIntegrityCheckSupported(); @@ -319,7 +330,9 @@ public void listAllInSortedOrderAsync( int limit, ActionListener> listener ) { - threadPool.executor(threadpoolName).execute(() -> { listAllInSortedOrder(path, filenamePrefix, limit, listener); }); + threadPool.executor(threadpoolName).execute(() -> { + listAllInSortedOrder(path, filenamePrefix, limit, listener); + }); } } From 5c3e28b9459f04f5d83c5d85a854889715311464 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 7 Jun 2024 14:44:39 +0530 Subject: [PATCH 122/133] Use InputStreams rather than XContent for serialization Signed-off-by: Shivansh Arora --- .../RemoteClusterStateAttributesManager.java | 23 ++++++-- .../remote/RemoteClusterStateService.java | 23 +++----- .../remote/model/RemoteClusterBlocks.java | 16 +++--- .../model/RemoteClusterStateCustoms.java | 53 +++++++++++++------ .../remote/model/RemoteDiscoveryNodes.java | 16 +++--- .../RemoteHashesOfConsistentSettings.java | 16 +++--- .../main/java/org/opensearch/node/Node.java | 3 +- 7 files changed, 93 insertions(+), 57 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 4a4b0c79b21a9..e7d497bf2ee83 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.CheckedRunnable; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; @@ -45,14 +46,22 @@ public class RemoteClusterStateAttributesManager { private final RemoteClusterStateBlobStore customsBlobStore; private final Compressor compressor; private final NamedXContentRegistry namedXContentRegistry; + private final NamedWriteableRegistry namedWriteableRegistry; RemoteClusterStateAttributesManager( - RemoteClusterStateBlobStore clusterBlocksBlobStore, RemoteClusterStateBlobStore discoveryNodesBlobStore, RemoteClusterStateBlobStore customsBlobStore, Compressor compressor, NamedXContentRegistry namedXContentRegistry) { + RemoteClusterStateBlobStore clusterBlocksBlobStore, + RemoteClusterStateBlobStore discoveryNodesBlobStore, + RemoteClusterStateBlobStore customsBlobStore, + Compressor compressor, + NamedXContentRegistry namedXContentRegistry, + NamedWriteableRegistry namedWriteableRegistry + ) { this.clusterBlocksBlobStore = clusterBlocksBlobStore; this.discoveryNodesBlobStore = discoveryNodesBlobStore; this.customsBlobStore = customsBlobStore; this.compressor = compressor; this.namedXContentRegistry = namedXContentRegistry; + this.namedWriteableRegistry = namedWriteableRegistry; } /** @@ -77,7 +86,8 @@ CheckedRunnable getAsyncMetadataWriteAction( clusterState.version(), clusterState.metadata().clusterUUID(), compressor, - namedXContentRegistry + namedXContentRegistry, + namedWriteableRegistry ); return () -> customsBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else { @@ -110,7 +120,14 @@ public CheckedRunnable getAsyncMetadataReadAction( return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); } else if (component.equals(CLUSTER_STATE_CUSTOM)) { final ActionListener customActionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, String.join(CUSTOM_DELIMITER, component, componentName))), listener::onFailure); - RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(uploadedFilename, componentName, clusterUUID, compressor, namedXContentRegistry); + RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms( + uploadedFilename, + componentName, + clusterUUID, + compressor, + namedXContentRegistry, + namedWriteableRegistry + ); return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, customActionListener); } else { throw new RemoteStateTransferException("Remote object not found for "+ component); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 26b1d0c4de98f..2918317622038 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -76,6 +76,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata; @@ -153,6 +154,7 @@ public class RemoteClusterStateService implements Closeable { private RemoteClusterStateAttributesManager remoteClusterStateAttributesManager; private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; + private NamedWriteableRegistry namedWriteableRegistry; private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " @@ -169,11 +171,6 @@ public class RemoteClusterStateService implements Closeable { FORMAT_PARAMS = new ToXContent.MapParams(params); } - private String latestClusterName; - private String latestClusterUUID; - private long lastCleanupAttemptState; - private boolean isClusterManagerNode; - public RemoteClusterStateService( String nodeId, Supplier repositoriesService, @@ -181,7 +178,8 @@ public RemoteClusterStateService( ClusterService clusterService, LongSupplier relativeTimeNanosSupplier, ThreadPool threadPool, - List indexMetadataUploadListeners + List indexMetadataUploadListeners, + NamedWriteableRegistry namedWriteableRegistry ) { assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled"; logger.info("REMOTE STATE ENABLED"); @@ -197,12 +195,10 @@ public RemoteClusterStateService( this.remoteStateReadTimeout = clusterSettings.get(REMOTE_STATE_READ_TIMEOUT_SETTING); clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteClusterStateEnabled); this.remoteStateStats = new RemotePersistenceStats(); - + this.namedWriteableRegistry = namedWriteableRegistry; this.remoteRoutingTableService = isRemoteRoutingTableEnabled(settings) ? Optional.of(new RemoteRoutingTableService(repositoriesService, settings, threadPool)) : Optional.empty(); - this.lastCleanupAttemptState = 0; - this.isClusterManagerNode = DiscoveryNode.isClusterManagerNode(settings); this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService); this.indexMetadataUploadListeners = indexMetadataUploadListeners; } @@ -439,9 +435,6 @@ public ClusterMetadataManifest writeIncrementalMetadata( logger.info("MANIFEST IN INC STATE {}", manifest); - this.latestClusterName = clusterState.getClusterName().value(); - this.latestClusterUUID = clusterState.metadata().clusterUUID(); - final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); @@ -904,8 +897,7 @@ public void start() { RemoteClusterStateBlobStore clusterStateCustomsBlobStore = new RemoteClusterStateBlobStore<>( getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); remoteClusterStateAttributesManager = new RemoteClusterStateAttributesManager(clusterBlocksBlobStore, discoveryNodesBlobStore, - clusterStateCustomsBlobStore, - blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry()); + clusterStateCustomsBlobStore, blobStoreRepository.getCompressor(), blobStoreRepository.getNamedXContentRegistry(), namedWriteableRegistry); RemoteClusterStateBlobStore manifestBlobStore = new RemoteClusterStateBlobStore<>( getBlobStoreTransferService(), blobStoreRepository, clusterName, threadpool, ThreadPool.Names.GENERIC); remoteManifestManager = new RemoteManifestManager(manifestBlobStore, clusterSettings, nodeId, blobStoreRepository.getCompressor(), @@ -958,7 +950,6 @@ public ClusterState getLatestClusterState(String clusterName, String clusterUUID private ClusterState readClusterStateInParallel( ClusterState previousState, ClusterMetadataManifest manifest, - String clusterName, String clusterUUID, String localNodeId, List indicesToRead, @@ -1233,7 +1224,6 @@ public ClusterState getClusterStateForManifest(String clusterName, ClusterMetada return readClusterStateInParallel( ClusterState.builder(new ClusterName(clusterName)).build(), manifest, - clusterName, manifest.getClusterUUID(), localNodeId, manifest.getIndices(), @@ -1279,7 +1269,6 @@ public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadata ClusterState updatedClusterState = readClusterStateInParallel( previousState, manifest, - clusterName, manifest.getClusterUUID(), localNodeId, updatedIndices, diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index c2fff1ee15439..707c4d54e04fa 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote.model; +import static org.opensearch.core.common.bytes.BytesReference.toBytes; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; @@ -17,8 +18,11 @@ import java.util.List; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; @@ -33,11 +37,6 @@ public class RemoteClusterBlocks extends AbstractRemoteWritableBlobEntity { public static final String CLUSTER_BLOCKS = "blocks"; - public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( - "blocks", - METADATA_NAME_FORMAT, - ClusterBlocks::fromXContent - ); private ClusterBlocks clusterBlocks; private long stateVersion; @@ -81,11 +80,14 @@ public UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - return CLUSTER_BLOCKS_FORMAT.serialize(clusterBlocks, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + clusterBlocks.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); } @Override public ClusterBlocks deserialize(final InputStream inputStream) throws IOException { - return CLUSTER_BLOCKS_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream))); + return ClusterBlocks.readFrom(in); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java index 58fbbdad3c4b5..b5af4969b7454 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -10,9 +10,15 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; @@ -24,6 +30,8 @@ import java.io.InputStream; import java.util.List; +import static org.opensearch.cluster.ClusterState.FeatureAware.shouldSerialize; +import static org.opensearch.core.common.bytes.BytesReference.toBytes; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; @@ -33,32 +41,39 @@ public class RemoteClusterStateCustoms extends AbstractRemoteWritableBlobEntity { public static final String CLUSTER_STATE_CUSTOM = "cluster-state-custom"; - public final ChecksumBlobStoreFormat clusterStateCustomBlobStoreFormat; private long stateVersion; private String customType; private ClusterState.Custom custom; + private final NamedWriteableRegistry namedWriteableRegistry; - public RemoteClusterStateCustoms(final ClusterState.Custom custom, final String customType, final long stateVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteClusterStateCustoms( + final ClusterState.Custom custom, + final String customType, + final long stateVersion, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry, + final NamedWriteableRegistry namedWriteableRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.stateVersion = stateVersion; this.customType = customType; this.custom = custom; - this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( - CLUSTER_STATE_CUSTOM, - METADATA_NAME_FORMAT, - parser -> ClusterState.Custom.fromXContent(parser, customType) - ); + this.namedWriteableRegistry = namedWriteableRegistry; } - public RemoteClusterStateCustoms(final String blobName, final String customType, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteClusterStateCustoms( + final String blobName, + final String customType, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry, + final NamedWriteableRegistry namedWriteableRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; this.customType = customType; - this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( - CLUSTER_STATE_CUSTOM, - METADATA_NAME_FORMAT, - parser -> ClusterState.Custom.fromXContent(parser, customType) - ); + this.namedWriteableRegistry = namedWriteableRegistry; } @Override @@ -88,11 +103,19 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - return clusterStateCustomBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput outputStream = new BytesStreamOutput(); + if (shouldSerialize(outputStream, custom)) { + outputStream.writeNamedWriteable(custom); + } + return outputStream.bytes().streamInput(); } @Override public ClusterState.Custom deserialize(final InputStream inputStream) throws IOException { - return clusterStateCustomBlobStoreFormat.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput( + new BytesStreamInput(toBytes(Streams.readFully(inputStream))), + this.namedWriteableRegistry + ); + return in.readNamedWriteable(Custom.class); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index e63e6ed63f6d1..d5f2bcac3f2a9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -8,18 +8,23 @@ package org.opensearch.gateway.remote.model; +import static org.opensearch.core.common.bytes.BytesReference.toBytes; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.List; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; @@ -34,11 +39,6 @@ public class RemoteDiscoveryNodes extends AbstractRemoteWritableBlobEntity { public static final String DISCOVERY_NODES = "nodes"; - public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( - "nodes", - METADATA_NAME_FORMAT, - DiscoveryNodes::fromXContent - ); private DiscoveryNodes discoveryNodes; private long stateVersion; @@ -81,11 +81,13 @@ public UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - return DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput outputStream = new BytesStreamOutput(); + discoveryNodes.writeTo(outputStream); + return outputStream.bytes().streamInput(); } @Override public DiscoveryNodes deserialize(final InputStream inputStream) throws IOException { - return DISCOVERY_NODES_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + return DiscoveryNodes.readFrom(new BytesStreamInput(toBytes(Streams.readFully(inputStream))), null); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java index d7908ac6e8235..f8ef85616a578 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -8,6 +8,7 @@ package org.opensearch.gateway.remote.model; +import static org.opensearch.core.common.bytes.BytesReference.toBytes; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; @@ -18,8 +19,11 @@ import java.util.List; import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; @@ -29,11 +33,6 @@ public class RemoteHashesOfConsistentSettings extends AbstractRemoteWritableBlobEntity { public static final String HASHES_OF_CONSISTENT_SETTINGS = "hashes-of-consistent-settings"; - public static final ChecksumBlobStoreFormat HASHES_OF_CONSISTENT_SETTINGS_FORMAT = new ChecksumBlobStoreFormat<>( - HASHES_OF_CONSISTENT_SETTINGS, - METADATA_NAME_FORMAT, - DiffableStringMap::fromXContent - ); private DiffableStringMap hashesOfConsistentSettings; private long metadataVersion; @@ -74,11 +73,14 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.serialize(hashesOfConsistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + hashesOfConsistentSettings.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); } @Override public DiffableStringMap deserialize(final InputStream inputStream) throws IOException { - return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream))); + return DiffableStringMap.readFrom(in); } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index c2866247fd723..6d5e47404a5b3 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -769,7 +769,8 @@ protected Node( clusterService, threadPool::preciseRelativeTimeInNanos, threadPool, - List.of(remoteIndexPathUploader) + List.of(remoteIndexPathUploader), + namedWriteableRegistry ); remoteClusterStateCleanupManager = remoteClusterStateService.getCleanupManager(); } else { From 3147aa35d72b1c6d0f0d96a6fc5b81082b593349 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 7 Jun 2024 15:35:07 +0530 Subject: [PATCH 123/133] Add Remote Entity implementation unit tests --- .../RemoteClusterStateCleanupManager.java | 4 +- .../gateway/remote/RemoteManifestManager.java | 6 +- .../model/RemoteClusterMetadataManifest.java | 9 +- ...RemoteClusterStateCleanupManagerTests.java | 9 +- .../RemoteClusterMetadataManifestTests.java | 35 +++- .../RemoteCoordinationMetadataTests.java | 178 +++++++++++++++++ .../model/RemoteCustomMetadataTests.java | 181 +++++++++++++++++ .../model/RemoteGlobalMetadataTests.java | 162 +++++++++++++++ .../RemotePersistentSettingsMetadataTest.java | 172 ++++++++++++++++ .../model/RemoteTemplatesMetadataTest.java | 184 ++++++++++++++++++ 10 files changed, 915 insertions(+), 25 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTest.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java index 6e3e665d8bb38..f650678358065 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManager.java @@ -34,7 +34,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST; /** * A Manager which provides APIs to clean up stale cluster state files and runs an async stale cleanup task @@ -255,7 +255,7 @@ void deleteStaleClusterMetadata(String clusterName, String clusterUUID, int mani getBlobStoreTransferService().listAllInSortedOrderAsync( ThreadPool.Names.REMOTE_PURGE, remoteClusterStateService.getRemoteManifestManager().getManifestFolderPath(clusterName, clusterUUID), - MANIFEST_FILE_PREFIX, + MANIFEST, Integer.MAX_VALUE, new ActionListener<>() { @Override diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index e6cbdcb5c31de..544191d9ab324 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -214,7 +214,7 @@ private BlobContainer manifestContainer(String clusterName, String clusterUUID) } BlobPath getManifestFolderPath(String clusterName, String clusterUUID) { - return getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN); + return getCusterMetadataBasePath(blobStoreRepository, clusterName, clusterUUID).add(RemoteClusterMetadataManifest.MANIFEST); } public TimeValue getMetadataManifestUploadTimeout() { @@ -254,7 +254,7 @@ private List getManifestFileNames(String clusterName, String clust static String getManifestFilePrefixForTermVersion(long term, long version) { return String.join( DELIMITER, - RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX, + RemoteClusterMetadataManifest.MANIFEST, RemoteStoreUtils.invertLong(term), RemoteStoreUtils.invertLong(version) ) + DELIMITER; @@ -268,7 +268,7 @@ static String getManifestFilePrefixForTermVersion(long term, long version) { * @return latest ClusterMetadataManifest filename */ private Optional getLatestManifestFileName(String clusterName, String clusterUUID) throws IllegalStateException { - List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX + DELIMITER, 1); + List manifestFilesMetadata = getManifestFileNames(clusterName, clusterUUID, RemoteClusterMetadataManifest.MANIFEST + DELIMITER, 1); if (manifestFilesMetadata != null && !manifestFilesMetadata.isEmpty()) { return Optional.of(manifestFilesMetadata.get(0).name()); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index 9343dea21fbe0..d8365bae538f8 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -30,10 +30,9 @@ */ public class RemoteClusterMetadataManifest extends AbstractRemoteWritableBlobEntity { - public static final String MANIFEST_PATH_TOKEN = "manifest"; + public static final String MANIFEST = "manifest"; public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; - public static final String MANIFEST_FILE_PREFIX = "manifest"; public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; @@ -73,7 +72,7 @@ public RemoteClusterMetadataManifest(final String blobName, final String cluster @Override public BlobPathParameters getBlobPathParameters() { - return new BlobPathParameters(List.of(MANIFEST_PATH_TOKEN), MANIFEST_FILE_PREFIX); + return new BlobPathParameters(List.of(MANIFEST), MANIFEST); } @Override @@ -82,7 +81,7 @@ public String generateBlobFileName() { // String blobFileName = String.join( DELIMITER, - MANIFEST_FILE_PREFIX, + MANIFEST, RemoteStoreUtils.invertLong(clusterMetadataManifest.getClusterTerm()), RemoteStoreUtils.invertLong(clusterMetadataManifest.getStateVersion()), (clusterMetadataManifest.isCommitted() ? "C" : "P"), // C for committed and P for published @@ -97,7 +96,7 @@ public String generateBlobFileName() { @Override public UploadedMetadata getUploadedMetadata() { assert blobName != null; - return new UploadedMetadataAttribute(MANIFEST_PATH_TOKEN, blobName); + return new UploadedMetadataAttribute(MANIFEST, blobName); } @Override diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java index 795fcc20703dd..e602748e45346 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateCleanupManagerTests.java @@ -63,12 +63,11 @@ import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.encodeString; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA; import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA; import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; -import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -221,7 +220,7 @@ public void testDeleteClusterMetadata() throws IOException { List.of(new BlobPath().add(INDEX_PATH_TOKEN).add(index1Metadata.getIndexUUID()).buildAsString() + index1Metadata.getUploadedFilePath() + ".dat") ); Set staleManifest = new HashSet<>(); - inactiveBlobs.forEach(blob -> staleManifest.add(new BlobPath().add(MANIFEST_PATH_TOKEN).buildAsString() + blob.name())); + inactiveBlobs.forEach(blob -> staleManifest.add(new BlobPath().add(MANIFEST).buildAsString() + blob.name())); verify(container).deleteBlobsIgnoringIfNotExists(new ArrayList<>(staleManifest)); } @@ -257,14 +256,14 @@ public void testDeleteStaleClusterUUIDs() throws IOException { }); when( manifest2Container.listBlobsByPrefixInSortedOrder( - MANIFEST_FILE_PREFIX + DELIMITER, + MANIFEST + DELIMITER, Integer.MAX_VALUE, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC ) ).thenReturn(List.of(new PlainBlobMetadata("mainfest2", 1L))); when( manifest3Container.listBlobsByPrefixInSortedOrder( - MANIFEST_FILE_PREFIX + DELIMITER, + MANIFEST + DELIMITER, Integer.MAX_VALUE, BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC ) diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java index cd852731be21b..7c5c8aa105b44 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java @@ -16,9 +16,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_FILE_PREFIX; -import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Collections; @@ -100,7 +100,8 @@ public void testClusterUUID() { RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); - RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); } @@ -109,7 +110,8 @@ public void testFullBlobName() { RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); - RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); } @@ -118,16 +120,23 @@ public void testBlobFileName() { RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); - RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); } + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/manifest"; + RemoteClusterMetadataManifest remoteObjectForDownload = new RemoteClusterMetadataManifest(uploadedFile, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[]{"user", "local", "opensearch", "manifest"})); + } + public void testBlobPathParameters() { ClusterMetadataManifest manifest = getClusterMetadataManifest(); RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); - assertThat(params.getPathTokens(), is(List.of(MANIFEST_PATH_TOKEN))); - assertThat(params.getFilePrefix(), is(MANIFEST_FILE_PREFIX)); + assertThat(params.getPathTokens(), is(List.of(MANIFEST))); + assertThat(params.getFilePrefix(), is(RemoteClusterMetadataManifest.MANIFEST)); } public void testGenerateBlobFileName() { @@ -135,7 +144,7 @@ public void testGenerateBlobFileName() { RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(manifest, clusterUUID, compressor, namedXContentRegistry); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); - assertThat(nameTokens[0], is(MANIFEST_FILE_PREFIX)); + assertThat(nameTokens[0], is(RemoteClusterMetadataManifest.MANIFEST)); assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(TERM)); assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), is(VERSION)); assertThat(nameTokens[3], is("C")); @@ -151,7 +160,7 @@ public void testGetUploadedMetadata() throws IOException { try (InputStream inputStream = remoteObjectForUpload.serialize()) { remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); - assertThat(uploadedMetadata.getComponent(), is(MANIFEST_PATH_TOKEN)); + assertThat(uploadedMetadata.getComponent(), is(MANIFEST)); assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); } } @@ -165,6 +174,10 @@ public void testSerDe() throws IOException { ClusterMetadataManifest readManifest = remoteObjectForUpload.deserialize(inputStream); assertThat(readManifest, is(manifest)); } + + String blobName = "/usr/local/manifest__1__2__3__4__5__6"; + RemoteClusterMetadataManifest invalidRemoteObject = new RemoteClusterMetadataManifest(blobName, clusterUUID, compressor, namedXContentRegistry); + assertThrows(IllegalArgumentException.class, () -> invalidRemoteObject.deserialize(new ByteArrayInputStream(new byte[0]))); } private ClusterMetadataManifest getClusterMetadataManifest() { @@ -174,6 +187,8 @@ private ClusterMetadataManifest getClusterMetadataManifest() { ClusterStateDiffManifest.builder().fromStateUUID("from-uuid").toStateUUID("to-uuid").indicesUpdated(Collections.emptyList()) .indicesDeleted(Collections.emptyList()) .customMetadataUpdated(Collections.emptyList()).customMetadataDeleted(Collections.emptyList()) - .indicesRoutingUpdated(Collections.emptyList()).indicesRoutingDeleted(Collections.emptyList()).build()).build(); + .indicesRoutingUpdated(Collections.emptyList()).indicesRoutingDeleted(Collections.emptyList()) + .clusterStateCustomUpdated(Collections.emptyList()).clusterStateCustomDeleted(Collections.emptyList()).build()) + .build(); } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java new file mode 100644 index 0000000000000..90be9abc5cc5d --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; +import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteCoordinationMetadataTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long TERM = 3L; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testClusterUUID() { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(); + RemoteCoordinationMetadata remoteObjectForUpload = new RemoteCoordinationMetadata(coordinationMetadata, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteCoordinationMetadata remoteObjectForDownload = new RemoteCoordinationMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(); + RemoteCoordinationMetadata remoteObjectForUpload = new RemoteCoordinationMetadata(coordinationMetadata, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemoteCoordinationMetadata remoteObjectForDownload = new RemoteCoordinationMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(); + RemoteCoordinationMetadata remoteObjectForUpload = new RemoteCoordinationMetadata(coordinationMetadata, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteCoordinationMetadata remoteObjectForDownload = new RemoteCoordinationMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/coordinationMetadata"; + RemoteCoordinationMetadata remoteObjectForDownload = new RemoteCoordinationMetadata(uploadedFile, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[]{"user", "local", "opensearch", "coordinationMetadata"})); + } + + public void testBlobPathParameters() { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(); + RemoteCoordinationMetadata remoteObjectForUpload = new RemoteCoordinationMetadata(coordinationMetadata, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN))); + assertThat(params.getFilePrefix(), is(RemoteCoordinationMetadata.COORDINATION_METADATA)); + } + + public void testGenerateBlobFileName() { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(); + RemoteCoordinationMetadata remoteObjectForUpload = new RemoteCoordinationMetadata(coordinationMetadata, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertThat(nameTokens[0], is(RemoteCoordinationMetadata.COORDINATION_METADATA)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(METADATA_VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION))); + + } + + public void testGetUploadedMetadata() throws IOException { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(); + RemoteCoordinationMetadata remoteObjectForUpload = new RemoteCoordinationMetadata(coordinationMetadata, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertThat(uploadedMetadata.getComponent(), is(RemoteCoordinationMetadata.COORDINATION_METADATA)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + CoordinationMetadata coordinationMetadata = getCoordinationMetadata(); + RemoteCoordinationMetadata remoteObjectForUpload = new RemoteCoordinationMetadata(coordinationMetadata, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertThat(inputStream.available(), greaterThan(0)); + CoordinationMetadata readcoordinationMetadata = remoteObjectForUpload.deserialize(inputStream); + assertThat(readcoordinationMetadata, is(coordinationMetadata)); + } + } + + private CoordinationMetadata getCoordinationMetadata() { + return CoordinationMetadata.builder().term(TERM).lastAcceptedConfiguration(new VotingConfiguration(Set.of("node1"))) + .lastCommittedConfiguration(new VotingConfiguration(Set.of("node1"))) + .addVotingConfigExclusion(new VotingConfigExclusion("node2", " node-2")) + .build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java new file mode 100644 index 0000000000000..4b3ef88680afc --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java @@ -0,0 +1,181 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.metadata.IndexGraveyard; +import org.opensearch.cluster.metadata.Metadata.Custom; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteCustomMetadataTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final String CUSTOM_TYPE = "test-custom"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); +// namedXContentRegistry = new NamedXContentRegistry(List.of(new Entry(Metadata.Custom.class, new ParseField(CUSTOM_TYPE), p->TestCustomMetadata.fromXContent(CustomMetadata1::new, p)))); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testClusterUUID() { + Custom customMetadata = getCustomMetadata(); + RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(customMetadata, "test-custom", METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteCustomMetadata remoteObjectForDownload = new RemoteCustomMetadata(TEST_BLOB_NAME, "test-custom", clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + Custom customMetadata = getCustomMetadata(); + RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(customMetadata, "test-custom", METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemoteCustomMetadata remoteObjectForDownload = new RemoteCustomMetadata(TEST_BLOB_NAME, "test-custom", clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + Custom customMetadata = getCustomMetadata(); + RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(customMetadata, "test-custom", METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteCustomMetadata remoteObjectForDownload = new RemoteCustomMetadata(TEST_BLOB_NAME, "test-custom", clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/customMetadata"; + RemoteCustomMetadata remoteObjectForDownload = new RemoteCustomMetadata(uploadedFile, "test-custom", clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[]{"user", "local", "opensearch", "customMetadata"})); + } + + public void testBlobPathParameters() { + Custom customMetadata = getCustomMetadata(); + RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(customMetadata, "test-custom", METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN))); + String expectedPrefix = CUSTOM_METADATA + CUSTOM_DELIMITER + CUSTOM_TYPE; + assertThat(params.getFilePrefix(), is(expectedPrefix)); + } + + public void testGenerateBlobFileName() { + Custom customMetadata = getCustomMetadata(); + RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(customMetadata, "test-custom", METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + String expectedPrefix = CUSTOM_METADATA + CUSTOM_DELIMITER + CUSTOM_TYPE; + assertThat(nameTokens[0], is(expectedPrefix)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(METADATA_VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION))); + + } + + public void testGetUploadedMetadata() throws IOException { + Custom customMetadata = getCustomMetadata(); + RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(customMetadata, "test-custom", METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + String expectedPrefix = CUSTOM_METADATA + CUSTOM_DELIMITER + CUSTOM_TYPE; + assertThat(uploadedMetadata.getComponent(), is(expectedPrefix)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + Custom customMetadata = getCustomMetadata(); + RemoteCustomMetadata remoteObjectForUpload = new RemoteCustomMetadata(customMetadata, IndexGraveyard.TYPE, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertThat(inputStream.available(), greaterThan(0)); + Custom readCustomMetadata = remoteObjectForUpload.deserialize(inputStream); + assertThat(readCustomMetadata, is(customMetadata)); + } + } + + private Custom getCustomMetadata() { + return IndexGraveyard.builder().addTombstone(new Index("test-index", "3q2423")).build(); + } + +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java new file mode 100644 index 0000000000000..3f5a1d1885d25 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteGlobalMetadataTests.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.model.RemoteGlobalMetadata.GLOBAL_METADATA_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfigExclusion; +import org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteGlobalMetadataTests extends OpenSearchTestCase { + + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long TERM = 3L; + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testClusterUUID() { + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/globalMetadata"; + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(uploadedFile, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[]{"user", "local", "opensearch", "globalMetadata"})); + } + + public void testBlobPathParameters() { + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); + assertThrows(UnsupportedOperationException.class, remoteObjectForDownload::getBlobPathParameters); + } + + public void testGenerateBlobFileName() { + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); + assertThrows(UnsupportedOperationException.class, remoteObjectForDownload::generateBlobFileName); + } + + public void testGetUploadedMetadata() { + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); + assertThrows(UnsupportedOperationException.class, remoteObjectForDownload::getUploadedMetadata); + } + + public void testSerDe() throws IOException { + Metadata globalMetadata = getGlobalMetadata(); + try (InputStream inputStream = GLOBAL_METADATA_FORMAT.serialize(globalMetadata, TEST_BLOB_FILE_NAME, compressor, RemoteClusterStateUtils.FORMAT_PARAMS) + .streamInput()) { + RemoteGlobalMetadata remoteObjectForDownload = new RemoteGlobalMetadata(TEST_BLOB_NAME, clusterUUID, compressor, namedXContentRegistry); + assertThat(inputStream.available(), greaterThan(0)); + Metadata readglobalMetadata = remoteObjectForDownload.deserialize(inputStream); + assertTrue(Metadata.isGlobalStateEquals(readglobalMetadata, globalMetadata)); + } + } + + private Metadata getGlobalMetadata() { + return Metadata.builder() + .templates( + TemplatesMetadata.builder() + .put( + IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .patterns(Arrays.asList("bar-*", "foo-*")) + .settings( + Settings.builder() + .put("index.random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)) + .build() + ) + .build() + ) + .build() + ) + .coordinationMetadata(CoordinationMetadata.builder().term(TERM).lastAcceptedConfiguration(new VotingConfiguration(Set.of("node1"))) + .lastCommittedConfiguration(new VotingConfiguration(Set.of("node1"))) + .addVotingConfigExclusion(new VotingConfigExclusion("node2", " node-2")) + .build()) + .build(); + + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java new file mode 100644 index 0000000000000..61e357a4f3204 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java @@ -0,0 +1,172 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemotePersistentSettingsMetadataTest extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testClusterUUID() { + Settings settings = getSettings(); + RemotePersistentSettingsMetadata remoteObjectForUpload = new RemotePersistentSettingsMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemotePersistentSettingsMetadata remoteObjectForDownload = new RemotePersistentSettingsMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + Settings settings = getSettings(); + RemotePersistentSettingsMetadata remoteObjectForUpload = new RemotePersistentSettingsMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemotePersistentSettingsMetadata remoteObjectForDownload = new RemotePersistentSettingsMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + Settings settings = getSettings(); + RemotePersistentSettingsMetadata remoteObjectForUpload = new RemotePersistentSettingsMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemotePersistentSettingsMetadata remoteObjectForDownload = new RemotePersistentSettingsMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/settings"; + RemotePersistentSettingsMetadata remoteObjectForDownload = new RemotePersistentSettingsMetadata(uploadedFile, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[]{"user", "local", "opensearch", "settings"})); + } + + public void testBlobPathParameters() { + Settings settings = getSettings(); + RemotePersistentSettingsMetadata remoteObjectForUpload = new RemotePersistentSettingsMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN))); + assertThat(params.getFilePrefix(), is(RemotePersistentSettingsMetadata.SETTING_METADATA)); + } + + public void testGenerateBlobFileName() { + Settings settings = getSettings(); + RemotePersistentSettingsMetadata remoteObjectForUpload = new RemotePersistentSettingsMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertThat(nameTokens[0], is(RemotePersistentSettingsMetadata.SETTING_METADATA)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(METADATA_VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION))); + + } + + public void testGetUploadedMetadata() throws IOException { + Settings settings = getSettings(); + RemotePersistentSettingsMetadata remoteObjectForUpload = new RemotePersistentSettingsMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertThat(uploadedMetadata.getComponent(), is(RemotePersistentSettingsMetadata.SETTING_METADATA)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + Settings settings = getSettings(); + RemotePersistentSettingsMetadata remoteObjectForUpload = new RemotePersistentSettingsMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertThat(inputStream.available(), greaterThan(0)); + Settings readsettings = remoteObjectForUpload.deserialize(inputStream); + assertThat(readsettings, is(settings)); + } + } + + private Settings getSettings() { + return Settings.builder() + .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)) + .build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTest.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTest.java new file mode 100644 index 0000000000000..fcdc08f264862 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTest.java @@ -0,0 +1,184 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static java.util.stream.Collectors.toList; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.indices.IndicesModule; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteTemplatesMetadataTest extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testClusterUUID() { + TemplatesMetadata settings = getTemplatesMetadata(); + RemoteTemplatesMetadata remoteObjectForUpload = new RemoteTemplatesMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteTemplatesMetadata remoteObjectForDownload = new RemoteTemplatesMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + TemplatesMetadata settings = getTemplatesMetadata(); + RemoteTemplatesMetadata remoteObjectForUpload = new RemoteTemplatesMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemoteTemplatesMetadata remoteObjectForDownload = new RemoteTemplatesMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + TemplatesMetadata settings = getTemplatesMetadata(); + RemoteTemplatesMetadata remoteObjectForUpload = new RemoteTemplatesMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteTemplatesMetadata remoteObjectForDownload = new RemoteTemplatesMetadata(TEST_BLOB_NAME, clusterUUID, compressor, + namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/settings"; + RemoteTemplatesMetadata remoteObjectForDownload = new RemoteTemplatesMetadata(uploadedFile, clusterUUID, compressor, namedXContentRegistry); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[]{"user", "local", "opensearch", "settings"})); + } + + public void testBlobPathParameters() { + TemplatesMetadata settings = getTemplatesMetadata(); + RemoteTemplatesMetadata remoteObjectForUpload = new RemoteTemplatesMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN))); + assertThat(params.getFilePrefix(), is(RemoteTemplatesMetadata.TEMPLATES_METADATA)); + } + + public void testGenerateBlobFileName() { + TemplatesMetadata settings = getTemplatesMetadata(); + RemoteTemplatesMetadata remoteObjectForUpload = new RemoteTemplatesMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertThat(nameTokens[0], is(RemoteTemplatesMetadata.TEMPLATES_METADATA)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(METADATA_VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION))); + + } + + public void testGetUploadedMetadata() throws IOException { + TemplatesMetadata settings = getTemplatesMetadata(); + RemoteTemplatesMetadata remoteObjectForUpload = new RemoteTemplatesMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertThat(uploadedMetadata.getComponent(), is(RemoteTemplatesMetadata.TEMPLATES_METADATA)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + TemplatesMetadata settings = getTemplatesMetadata(); + RemoteTemplatesMetadata remoteObjectForUpload = new RemoteTemplatesMetadata(settings, METADATA_VERSION, clusterUUID, compressor, namedXContentRegistry); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertThat(inputStream.available(), greaterThan(0)); + TemplatesMetadata readsettings = remoteObjectForUpload.deserialize(inputStream); + assertThat(readsettings, is(settings)); + } + } + + private TemplatesMetadata getTemplatesMetadata() { + return TemplatesMetadata.builder() + .put( + IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .patterns(Arrays.asList("bar-*", "foo-*")) + .settings( + Settings.builder() + .put("index.random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)) + .build() + ) + .build() + ) + .build(); + } +} From 074252303cf1c7ff33eb2e96bf6fdc0540bc0136 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 7 Jun 2024 15:35:07 +0530 Subject: [PATCH 124/133] Add unit tests for Remote Entity implementation Signed-off-by: Sooraj Sinha --- .../org/opensearch/gateway/remote/RemoteClusterStateUtils.java | 1 + .../gateway/remote/model/RemoteCoordinationMetadataTests.java | 2 +- .../gateway/remote/model/RemoteCustomMetadataTests.java | 2 +- .../remote/model/RemotePersistentSettingsMetadataTest.java | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 67280c15f1040..fbf355ca8a69c 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -32,6 +32,7 @@ public class RemoteClusterStateUtils { public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; public static final String CLUSTER_STATE_EPHEMERAL_PATH_TOKEN = "ephemeral"; + public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; public static final String DELIMITER = "__"; public static final String PATH_DELIMITER = "/"; diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java index 90be9abc5cc5d..97866d5fa16db 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCoordinationMetadataTests.java @@ -15,7 +15,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java index 4b3ef88680afc..b68409175305d 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteCustomMetadataTests.java @@ -15,7 +15,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java index 61e357a4f3204..f74110f7470fb 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java @@ -15,7 +15,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; import java.io.IOException; import java.io.InputStream; From 26a12cfffe20e096cc80e64ea4ed1dd4d0d43a78 Mon Sep 17 00:00:00 2001 From: Himshikha Gupta Date: Thu, 6 Jun 2024 14:53:19 +0530 Subject: [PATCH 125/133] Add publication flag and remote routing table check Signed-off-by: Himshikha Gupta --- .../coordination/CoordinationState.java | 8 +++++- .../cluster/coordination/Coordinator.java | 3 ++- .../PublicationTransportHandler.java | 25 +++++++++---------- .../coordination/RemotePublishRequest.java | 3 ++- .../cluster/node/DiscoveryNode.java | 11 +++++++- .../remote/RemoteRoutingTableService.java | 9 +++---- .../remote/RemoteClusterStateService.java | 5 ++-- .../remotestore/RemoteStoreNodeAttribute.java | 6 ++--- 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index d8e48279a0db6..6fe16b9b32ce9 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -50,6 +50,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import static org.opensearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; @@ -79,6 +80,7 @@ public class CoordinationState { private VotingConfiguration lastPublishedConfiguration; private VoteCollection publishVotes; private final boolean isRemoteStateEnabled; + private final boolean isRemotePublicationEnabled; public CoordinationState( DiscoveryNode localNode, @@ -102,12 +104,17 @@ public CoordinationState( .getLastAcceptedConfiguration(); this.publishVotes = new VoteCollection(); this.isRemoteStateEnabled = isRemoteStoreClusterStateEnabled(settings); + this.isRemotePublicationEnabled = RemoteStoreNodeAttribute.isRemotePublicationEnabled(settings); } public boolean isRemoteStateEnabled() { return isRemoteStateEnabled; } + public boolean isRemotePublicationEnabled() { + return isRemotePublicationEnabled; + } + public long getCurrentTerm() { return persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL).getCurrentTerm(); } @@ -453,7 +460,6 @@ public PublishResponse handlePublishRequest(PublishRequest publishRequest) { clusterState.version(), clusterState.term() ); - logger.info("Setting last accepted state : term - {}, version - {}", clusterState.term(), clusterState.version()); persistedStateRegistry.getPersistedState(PersistedStateType.LOCAL).setLastAcceptedState(clusterState); assert getLastAcceptedState() == clusterState; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 1b3d6aeb0f8f5..a9090686170d9 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -1323,7 +1323,8 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) + clusterState; final PublicationTransportHandler.PublicationContext publicationContext = publicationHandler.newPublicationContext( - clusterChangedEvent, coordinationState.get().isRemoteStateEnabled() + clusterChangedEvent, + coordinationState.get().isRemotePublicationEnabled() ); final PublishRequest publishRequest = coordinationState.get().handleClientValue(clusterState); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index bcad827517033..97bc2fc4e6d0e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -242,27 +242,27 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish boolean applyFullState = false; final ClusterState lastSeen = lastSeenClusterState.get(); if (lastSeen == null) { - logger.debug("Diff cannot be applied as there is no last cluster state"); + logger.debug(() -> "Diff cannot be applied as there is no last cluster state"); applyFullState = true; } else if (manifest.getDiffManifest() == null) { - logger.debug("There is no diff in the manifest"); + logger.trace(() -> "There is no diff in the manifest"); applyFullState = true; } else if (manifest.getDiffManifest().getFromStateUUID().equals(lastSeen.stateUUID()) == false) { - logger.debug("Last cluster state not compatible with the diff"); + logger.debug(() -> "Last cluster state not compatible with the diff"); applyFullState = true; } if (applyFullState == true) { - logger.info("Downloading full cluster state for term {}, version {}, stateUUID {}", manifest.getClusterTerm(), manifest.getStateVersion(), - manifest.getStateUUID()); + logger.info(() -> new ParameterizedMessage("Downloading full cluster state for term {}, version {}, stateUUID {}", manifest.getClusterTerm(), manifest.getStateVersion(), + manifest.getStateUUID())); ClusterState clusterState = remoteClusterStateService.getClusterStateForManifest(request.getClusterName(), manifest, transportService.getLocalNode().getId(), true); fullClusterStateReceivedCount.incrementAndGet(); final PublishWithJoinResponse response = acceptState(clusterState); lastSeenClusterState.set(clusterState); return response; } else { - logger.info("Downloading diff cluster state for term {}, version {}, previousUUID {}, current UUID {}", manifest.getClusterTerm(), - manifest.getStateVersion(), manifest.getDiffManifest().getFromStateUUID(), manifest.getStateUUID()); + logger.info(() -> new ParameterizedMessage("Downloading diff cluster state for term {}, version {}, previousUUID {}, current UUID {}", manifest.getClusterTerm(), + manifest.getStateVersion(), manifest.getDiffManifest().getFromStateUUID(), manifest.getStateUUID())); ClusterState clusterState = remoteClusterStateService.getClusterStateUsingDiff(request.getClusterName(), manifest, lastSeen, transportService.getLocalNode().getId()); compatibleClusterStateDiffReceivedCount.incrementAndGet(); final PublishWithJoinResponse response = acceptState(clusterState); @@ -285,7 +285,6 @@ private PublishWithJoinResponse acceptState(ClusterState incomingState) { } private PublishWithJoinResponse acceptStateOnLocalNode(RemotePublishRequest remotePublishRequest) { - // if the state is coming from the current node, use original request instead (see currentPublishRequestToSelf for explanation) final PublishRequest publishRequest = currentPublishRequestToSelf.get(); if (publishRequest == null || publishRequest.getAcceptedState().coordinationMetadata().term() != remotePublishRequest.term || publishRequest.getAcceptedState().version() != remotePublishRequest.version) { @@ -296,8 +295,8 @@ private PublishWithJoinResponse acceptStateOnLocalNode(RemotePublishRequest remo return publishWithJoinResponse; } - public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemoteStateEnabled) { - final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemoteStateEnabled); + public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled) { + final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemotePublicationEnabled); // Build the serializations we expect to need now, early in the process, so that an error during serialization fails the publication // straight away. This isn't watertight since we send diffs on a best-effort basis and may fall back to sending a full state (and @@ -344,12 +343,12 @@ public class PublicationContext { private final Map serializedDiffs = new HashMap<>(); private final boolean sendRemoteState; - PublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemoteStateEnabled) { + PublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled) { discoveryNodes = clusterChangedEvent.state().nodes(); newState = clusterChangedEvent.state(); previousState = clusterChangedEvent.previousState(); sendFullVersion = previousState.getBlocks().disableStatePersistence(); - sendRemoteState = isRemoteStateEnabled; + sendRemoteState = isRemotePublicationEnabled; } void buildDiffAndSerializeStates() { @@ -413,7 +412,7 @@ public void onFailure(Exception e) { } else { responseActionListener = listener; } - if (sendRemoteState && destination.isRemoteStateNode()) { + if (sendRemoteState && destination.isRemoteClusterStateEnabled() && destination.isRemoteRoutingTableEnabled()) { sendRemoteClusterState(destination, publishRequest.getAcceptedState(), responseActionListener); } else if (sendFullVersion || previousState.nodes().nodeExists(destination) == false) { logger.trace("sending full cluster state version [{}] to [{}]", newState.version(), destination); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java index e59f7cedd98e5..5e14548c1cc57 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java @@ -8,11 +8,12 @@ package org.opensearch.cluster.coordination; -import java.io.IOException; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; +import java.io.IOException; + public class RemotePublishRequest extends TermVersionRequest { // todo Do we need cluster name and UUID ? diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index fb27fccda0737..85c3e97ca71ad 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -67,6 +67,7 @@ import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; /** * A discovery node represents a node that is part of the cluster. @@ -487,13 +488,21 @@ public boolean isRemoteStoreNode() { * Returns whether the node is a remote cluster state enabled node. * @return true if the node contains remote cluster state node attribute, false otherwise */ - public boolean isRemoteStateNode() { + public boolean isRemoteClusterStateEnabled() { return this.getAttributes() .keySet() .stream() .anyMatch(key -> (key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY))); } + /** + * Returns whether remote routing table is enabled on the node + * @return true if the node contains remote routing table node attributes, false otherwise + */ + public boolean isRemoteRoutingTableEnabled() { + return this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY)); + } + /** * Returns a set of all the roles that the node has. The roles are returned in sorted order by the role name. *

diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index b2a2eb4401bc1..83c756c98af14 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -13,7 +13,6 @@ import org.apache.lucene.store.IndexInput; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.Diff; import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; @@ -48,7 +47,6 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.threadpool.ThreadPool; -import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -62,9 +60,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.opensearch.common.blobstore.transfer.RemoteTransferContainer.checksumOfChecksum; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.getCusterMetadataBasePath; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemotePublicationEnabled; /** * A Service which provides APIs to upload and download routing table from remote store. @@ -109,7 +106,7 @@ public IndexRoutingTable read(StreamInput in, String key) throws IOException { public RemoteRoutingTableService(Supplier repositoriesService, Settings settings, ThreadPool threadPool) { - assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; + assert isRemotePublicationEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; this.threadPool = threadPool; @@ -301,7 +298,7 @@ public void doClose() throws IOException { @Override protected void doStart() { - assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled"; + assert isRemotePublicationEnabled(settings) == true : "Remote routing table is not enabled"; final String remoteStoreRepo = settings.get( Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY ); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 2918317622038..c8e3c505ba8d6 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -24,7 +24,7 @@ import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; import static org.opensearch.gateway.remote.model.RemoteIndexMetadata.INDEX_PATH_TOKEN; import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemotePublicationEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; import java.io.Closeable; @@ -59,7 +59,6 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.Metadata.Custom; import org.opensearch.cluster.metadata.TemplatesMetadata; -import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; @@ -196,7 +195,7 @@ public RemoteClusterStateService( clusterSettings.addSettingsUpdateConsumer(REMOTE_STATE_READ_TIMEOUT_SETTING, this::setRemoteClusterStateEnabled); this.remoteStateStats = new RemotePersistenceStats(); this.namedWriteableRegistry = namedWriteableRegistry; - this.remoteRoutingTableService = isRemoteRoutingTableEnabled(settings) + this.remoteRoutingTableService = isRemotePublicationEnabled(settings) ? Optional.of(new RemoteRoutingTableService(repositoriesService, settings, threadPool)) : Optional.empty(); this.remoteClusterStateCleanupManager = new RemoteClusterStateCleanupManager(this, clusterService); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 9a8bcc419144e..8a7a27f6833b9 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -12,7 +12,6 @@ import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.gateway.remote.RemoteClusterStateService; @@ -202,8 +201,9 @@ private static boolean isRemoteRoutingTableAttributePresent(Settings settings) { .isEmpty() == false; } - public static boolean isRemoteRoutingTableEnabled(Settings settings) { - return FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) && isRemoteRoutingTableAttributePresent(settings); + public static boolean isRemotePublicationEnabled(Settings settings) { + return FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL) && isRemoteRoutingTableAttributePresent(settings) + && isRemoteStoreClusterStateEnabled(settings); } public RepositoriesMetadata getRepositoriesMetadata() { From 5b835997249dd6ca57691a501c9469264e5109ea Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 7 Jun 2024 17:32:33 +0530 Subject: [PATCH 126/133] Add manifest file name Signed-off-by: Sooraj Sinha --- .../coordination/CoordinationState.java | 4 -- .../cluster/coordination/Coordinator.java | 5 ++- .../PublicationTransportHandler.java | 35 +++++++++++------- .../coordination/RemotePublishRequest.java | 12 +++++- .../remote/RemoteClusterStateService.java | 37 +++++++++++++------ .../gateway/remote/RemoteManifestManager.java | 20 ++++++++-- 6 files changed, 76 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java index 6fe16b9b32ce9..ede89b6d04cfe 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/CoordinationState.java @@ -107,10 +107,6 @@ public CoordinationState( this.isRemotePublicationEnabled = RemoteStoreNodeAttribute.isRemotePublicationEnabled(settings); } - public boolean isRemoteStateEnabled() { - return isRemoteStateEnabled; - } - public boolean isRemotePublicationEnabled() { return isRemotePublicationEnabled; } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index a9090686170d9..5446aadb62b48 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -186,6 +186,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; private final RemoteStoreNodeService remoteStoreNodeService; + private final RemoteClusterStateService remoteClusterStateService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -300,6 +301,7 @@ public Coordinator( this.persistedStateRegistry = persistedStateRegistry; this.localNodeCommissioned = true; this.remoteStoreNodeService = remoteStoreNodeService; + this.remoteClusterStateService = remoteClusterStateService; } private ClusterFormationState getClusterFormationState() { @@ -1324,7 +1326,8 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) final PublicationTransportHandler.PublicationContext publicationContext = publicationHandler.newPublicationContext( clusterChangedEvent, - coordinationState.get().isRemotePublicationEnabled() + coordinationState.get().isRemotePublicationEnabled(), + remoteClusterStateService.getLastUploadedManifestFileName() ); final PublishRequest publishRequest = coordinationState.get().handleClientValue(clusterState); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index 97bc2fc4e6d0e..fb4893e46c345 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Optional; +import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -42,6 +43,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.Diff; import org.opensearch.cluster.IncompatibleClusterStateVersionException; +import org.opensearch.cluster.coordination.CoordinationState.PersistedState; +import org.opensearch.cluster.coordination.PersistedStateRegistry.PersistedStateType; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.core.action.ActionListener; @@ -49,6 +52,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.transport.TransportResponse; +import org.opensearch.gateway.GatewayMetaState.RemotePersistedState; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.threadpool.ThreadPool; @@ -232,13 +236,13 @@ private PublishWithJoinResponse handleIncomingRemotePublishRequest(RemotePublish if (transportService.getLocalNode().equals(request.getSourceNode())) { return acceptStateOnLocalNode(request); } - final Optional manifestOptional = remoteClusterStateService.getClusterMetadataManifestByTermVersion(request.getClusterName(), request.getClusterUUID(), request.term, request.version); - if (manifestOptional.isPresent() == false) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Manifest is not present for term - %s version - %s", request.term, request.version) - ); - } - ClusterMetadataManifest manifest = manifestOptional.get(); +// final Optional manifestOptional = remoteClusterStateService.getClusterMetadataManifestByTermVersion(request.getClusterName(), request.getClusterUUID(), request.term, request.version); +// if (manifestOptional.isPresent() == false) { +// throw new IllegalStateException( +// String.format(Locale.ROOT, "Manifest is not present for term - %s version - %s", request.term, request.version) +// ); +// } + ClusterMetadataManifest manifest = remoteClusterStateService.getClusterMetadataManifestByFileName(request.getClusterUUID(), request.getManifestFile()); boolean applyFullState = false; final ClusterState lastSeen = lastSeenClusterState.get(); if (lastSeen == null) { @@ -295,8 +299,9 @@ private PublishWithJoinResponse acceptStateOnLocalNode(RemotePublishRequest remo return publishWithJoinResponse; } - public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled) { - final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemotePublicationEnabled); + public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled, + String manifestFileName) { + final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemotePublicationEnabled, manifestFileName); // Build the serializations we expect to need now, early in the process, so that an error during serialization fails the publication // straight away. This isn't watertight since we send diffs on a best-effort basis and may fall back to sending a full state (and @@ -342,13 +347,15 @@ public class PublicationContext { private final Map serializedStates = new HashMap<>(); private final Map serializedDiffs = new HashMap<>(); private final boolean sendRemoteState; + private final String manifestFileName; - PublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled) { + PublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled, String manifestFileName) { discoveryNodes = clusterChangedEvent.state().nodes(); newState = clusterChangedEvent.state(); previousState = clusterChangedEvent.previousState(); sendFullVersion = previousState.getBlocks().disableStatePersistence(); sendRemoteState = isRemotePublicationEnabled; + this.manifestFileName = manifestFileName; } void buildDiffAndSerializeStates() { @@ -459,15 +466,15 @@ public String executor() { ); } - private void sendRemoteClusterState(DiscoveryNode destination, ClusterState clusterState, ActionListener listener) { + private void sendRemoteClusterState(final DiscoveryNode destination, final ClusterState clusterState, final ActionListener listener) { try { - final RemotePublishRequest remotePublishRequest = new RemotePublishRequest(discoveryNodes.getLocalNode(), clusterState.term(), clusterState.getVersion(), clusterState.getClusterName().value(), clusterState.metadata().clusterUUID()); + final RemotePublishRequest remotePublishRequest = new RemotePublishRequest(discoveryNodes.getLocalNode(), clusterState.term(), + clusterState.getVersion(), clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifestFileName); final Consumer transportExceptionHandler = exp -> { logger.debug(() -> new ParameterizedMessage("failed to send remote cluster state to {}", destination), exp); listener.onFailure(exp); }; - final TransportResponseHandler responseHandler = new TransportResponseHandler< - PublishWithJoinResponse>() { + final TransportResponseHandler responseHandler = new TransportResponseHandler<>() { @Override public PublishWithJoinResponse read(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java index 5e14548c1cc57..abbe1f0f78421 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java @@ -19,17 +19,20 @@ public class RemotePublishRequest extends TermVersionRequest { // todo Do we need cluster name and UUID ? private final String clusterName; private final String clusterUUID; + private final String manifestFile; - public RemotePublishRequest(DiscoveryNode sourceNode, long term, long version, String clusterName, String clusterUUID) { + public RemotePublishRequest(DiscoveryNode sourceNode, long term, long version, String clusterName, String clusterUUID, String manifestFile) { super(sourceNode, term, version); this.clusterName = clusterName; this.clusterUUID = clusterUUID; + this.manifestFile = manifestFile; } public RemotePublishRequest(StreamInput in) throws IOException { super(in); this.clusterName = in.readString(); this.clusterUUID = in.readString(); + this.manifestFile = in.readString(); } @Override @@ -37,12 +40,13 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(clusterName); out.writeString(clusterUUID); + out.writeString(manifestFile); } @Override public String toString() { return "RemotePublishRequest{" + "term=" + term + ", version=" + version + ", clusterName=" + clusterName + ", clusterUUID=" + clusterUUID - + ", sourceNode=" + sourceNode + '}'; + + ", sourceNode=" + sourceNode + ", manifestFile=" + manifestFile + '}'; } public String getClusterName() { @@ -52,4 +56,8 @@ public String getClusterName() { public String getClusterUUID() { return clusterUUID; } + + public String getManifestFile() { + return manifestFile; + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index c8e3c505ba8d6..5809ccb37b052 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -68,6 +68,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobStore; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -154,6 +155,7 @@ public class RemoteClusterStateService implements Closeable { private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; private NamedWriteableRegistry namedWriteableRegistry; + private String latestManifestFileName; private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " @@ -233,15 +235,16 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true ); - final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( + final Tuple manifestDetails = remoteManifestManager.uploadManifest( clusterState, uploadedMetadataResults, previousClusterUUID, new ClusterStateDiffManifest(clusterState, ClusterState.EMPTY_STATE), false ); + this.latestManifestFileName = manifestDetails.v2(); - logger.info("MANIFEST IN FULL STATE {}", manifest); + logger.info("MANIFEST IN FULL STATE {}", manifestDetails.v1()); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); @@ -259,7 +262,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedIndexMetadata.size() ); } - return manifest; + return manifestDetails.v1(); } /** @@ -424,22 +427,23 @@ public ClusterMetadataManifest writeIncrementalMetadata( uploadedMetadataResults.uploadedIndexMetadata = new ArrayList<>(allUploadedIndexMetadata.values()); uploadedMetadataResults.uploadedIndicesRoutingMetadata = allUploadedIndicesRouting; - final ClusterMetadataManifest manifest = remoteManifestManager.uploadManifest( + final Tuple manifestDetails = remoteManifestManager.uploadManifest( clusterState, uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), new ClusterStateDiffManifest(clusterState, previousClusterState), false ); + this.latestManifestFileName = manifestDetails.v2(); - logger.info("MANIFEST IN INC STATE {}", manifest); + logger.info("MANIFEST IN INC STATE {}", manifestDetails.v1()); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); ParameterizedMessage clusterStateUploadTimeMessage = new ParameterizedMessage( CLUSTER_STATE_UPLOAD_TIME_LOG_STRING, - manifest.getStateVersion(), + manifestDetails.v1().getStateVersion(), durationMillis ); ParameterizedMessage metadataUpdateMessage = new ParameterizedMessage( @@ -471,7 +475,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( "writing cluster state for version [{}] took [{}ms]; " + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", - manifest.getStateVersion(), + manifestDetails.v1().getStateVersion(), durationMillis, numIndicesUpdated, numIndicesUnchanged, @@ -481,7 +485,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( customsToUpload.size() ); } - return manifest; + return manifestDetails.v1(); } private UploadedMetadataResults writeMetadataInParallel( @@ -816,18 +820,23 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getClusterStateCustomMap() ); - ClusterMetadataManifest committedManifest = remoteManifestManager.uploadManifest( + Tuple committedManifestDetails = remoteManifestManager.uploadManifest( clusterState, uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), true ); - if (!previousManifest.isClusterUUIDCommitted() && committedManifest.isClusterUUIDCommitted()) { - remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifest); + this.latestManifestFileName = committedManifestDetails.v2(); + if (!previousManifest.isClusterUUIDCommitted() && committedManifestDetails.v1().isClusterUUIDCommitted()) { + remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifestDetails.v1()); } - return committedManifest; + return committedManifestDetails.v1(); + } + + public String getLastUploadedManifestFileName() { + return this.latestManifestFileName; } /** @@ -845,6 +854,10 @@ public Optional getClusterMetadataManifestByTermVersion return remoteManifestManager.getClusterMetadataManifestByTermVersion(clusterName, clusterUUID, term, version); } + public ClusterMetadataManifest getClusterMetadataManifestByFileName(String clusterUUID, String fileName) { + return remoteManifestManager.getRemoteClusterMetadataManifestByFileName(clusterUUID, fileName); + } + @Override public void close() throws IOException { remoteClusterStateCleanupManager.close(); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 544191d9ab324..7deb76fd8c0a0 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -30,6 +30,7 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; @@ -71,7 +72,7 @@ public class RemoteManifestManager { this.blobStoreRepository = blobStoreRepository; } - ClusterMetadataManifest uploadManifest( + Tuple uploadManifest( ClusterState clusterState, RemoteClusterStateUtils.UploadedMetadataResults uploadedMetadataResult, String previousClusterUUID, @@ -105,12 +106,12 @@ ClusterMetadataManifest uploadManifest( .clusterStateCustomMetadataMap(uploadedMetadataResult.uploadedClusterStateCustomMetadataMap) .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings); final ClusterMetadataManifest manifest = manifestBuilder.build(); - writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest); - return manifest; + String manifestFileName = writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest); + return new Tuple<>(manifest, manifestFileName); } } - private void writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest) { + private String writeMetadataManifest(String clusterName, String clusterUUID, ClusterMetadataManifest uploadManifest) { AtomicReference result = new AtomicReference(); AtomicReference exceptionReference = new AtomicReference(); @@ -147,6 +148,7 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust remoteClusterMetadataManifest.getBlobFileName(), uploadManifest.isCommitted() ? "commit" : "publish" ); + return remoteClusterMetadataManifest.getUploadedMetadata().getUploadedFilename(); } /** @@ -174,6 +176,16 @@ public Optional getClusterMetadataManifestByTermVersion return manifestFileName.map(s -> fetchRemoteClusterMetadataManifest(clusterName, clusterUUID, s)); } + public ClusterMetadataManifest getRemoteClusterMetadataManifestByFileName(String clusterUUID, String filename) + throws IllegalStateException { + try { + RemoteClusterMetadataManifest remoteClusterMetadataManifest = new RemoteClusterMetadataManifest(filename, clusterUUID, compressor, namedXContentRegistry); + return manifestBlobStore.read(remoteClusterMetadataManifest); + } catch (IOException e) { + throw new IllegalStateException(String.format(Locale.ROOT, "Error while downloading cluster metadata - %s", filename), e); + } + } + /** * Fetch ClusterMetadataManifest from remote state store * From e47d2852cdc2f49d3956d86776b5f5c862593a58 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 7 Jun 2024 18:22:53 +0530 Subject: [PATCH 127/133] Renaming test files Signed-off-by: Sooraj Sinha --- ...dataTest.java => RemotePersistentSettingsMetadataTests.java} | 2 +- ...latesMetadataTest.java => RemoteTemplatesMetadataTests.java} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename server/src/test/java/org/opensearch/gateway/remote/model/{RemotePersistentSettingsMetadataTest.java => RemotePersistentSettingsMetadataTests.java} (99%) rename server/src/test/java/org/opensearch/gateway/remote/model/{RemoteTemplatesMetadataTest.java => RemoteTemplatesMetadataTests.java} (99%) diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTests.java similarity index 99% rename from server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java rename to server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTests.java index f74110f7470fb..477027ab68ce8 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTest.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemotePersistentSettingsMetadataTests.java @@ -44,7 +44,7 @@ import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; -public class RemotePersistentSettingsMetadataTest extends OpenSearchTestCase { +public class RemotePersistentSettingsMetadataTests extends OpenSearchTestCase { private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; private static final String TEST_BLOB_PATH = "test-path"; private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTest.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java similarity index 99% rename from server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTest.java rename to server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java index fcdc08f264862..6ad99d5f9e8c3 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTest.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTemplatesMetadataTests.java @@ -47,7 +47,7 @@ import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; -public class RemoteTemplatesMetadataTest extends OpenSearchTestCase { +public class RemoteTemplatesMetadataTests extends OpenSearchTestCase { private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; private static final String TEST_BLOB_PATH = "test-path"; private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; From 79eb0899f54d1b3449b41fb8cca04d040a2cf316 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 7 Jun 2024 20:07:20 +0530 Subject: [PATCH 128/133] Send manifest file name Signed-off-by: Sooraj Sinha --- .../cluster/coordination/Coordinator.java | 6 +-- .../opensearch/gateway/GatewayMetaState.java | 18 ++++--- .../remote/RemoteClusterStateService.java | 39 ++++++--------- .../gateway/remote/RemoteManifestManager.java | 5 +- .../remote/model/RemoteUploadDetails.java | 30 +++++++++++ .../coordination/CoordinationStateTests.java | 4 +- .../GatewayMetaStatePersistedStateTests.java | 11 ++-- .../RemoteClusterStateServiceTests.java | 50 ++++++++++++------- 8 files changed, 106 insertions(+), 57 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteUploadDetails.java diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 5446aadb62b48..944f586dc3ff2 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -48,6 +48,7 @@ import org.opensearch.cluster.coordination.CoordinationState.VoteCollection; import org.opensearch.cluster.coordination.FollowersChecker.FollowerCheckRequest; import org.opensearch.cluster.coordination.JoinHelper.InitialJoinAccumulator; +import org.opensearch.cluster.coordination.PersistedStateRegistry.PersistedStateType; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; @@ -84,6 +85,7 @@ import org.opensearch.discovery.PeerFinder; import org.opensearch.discovery.SeedHostsProvider; import org.opensearch.discovery.SeedHostsResolver; +import org.opensearch.gateway.GatewayMetaState.RemotePersistedState; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; @@ -186,7 +188,6 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; private final RemoteStoreNodeService remoteStoreNodeService; - private final RemoteClusterStateService remoteClusterStateService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -301,7 +302,6 @@ public Coordinator( this.persistedStateRegistry = persistedStateRegistry; this.localNodeCommissioned = true; this.remoteStoreNodeService = remoteStoreNodeService; - this.remoteClusterStateService = remoteClusterStateService; } private ClusterFormationState getClusterFormationState() { @@ -1327,7 +1327,7 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) final PublicationTransportHandler.PublicationContext publicationContext = publicationHandler.newPublicationContext( clusterChangedEvent, coordinationState.get().isRemotePublicationEnabled(), - remoteClusterStateService.getLastUploadedManifestFileName() + ((RemotePersistedState) persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE)) ); final PublishRequest publishRequest = coordinationState.get().handleClientValue(clusterState); diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index c3056276706a0..ffa0c8392876b 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -64,6 +64,7 @@ import org.opensearch.env.NodeMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.model.RemoteUploadDetails; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult; import org.opensearch.node.Node; @@ -665,6 +666,7 @@ public static class RemotePersistedState implements PersistedState { private ClusterState lastAcceptedState; private ClusterMetadataManifest lastAcceptedManifest; + private String lastUploadedManifestFile; private final RemoteClusterStateService remoteClusterStateService; private String previousClusterUUID; @@ -693,7 +695,7 @@ public void setCurrentTerm(long currentTerm) { @Override public void setLastAcceptedState(ClusterState clusterState) { try { - final ClusterMetadataManifest manifest; + final RemoteUploadDetails manifestDetails; if (shouldWriteFullClusterState(clusterState)) { final Optional latestManifest = remoteClusterStateService.getLatestClusterMetadataManifest( clusterState.getClusterName().value(), @@ -711,15 +713,16 @@ public void setLastAcceptedState(ClusterState clusterState) { clusterState.metadata().clusterUUID() ); } - manifest = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID); + manifestDetails = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID); } else { assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) == true : "Previous manifest and previous ClusterState are not in sync"; - manifest = remoteClusterStateService.writeIncrementalMetadata(lastAcceptedState, clusterState, lastAcceptedManifest); + manifestDetails = remoteClusterStateService.writeIncrementalMetadata(lastAcceptedState, clusterState, lastAcceptedManifest); } - assert verifyManifestAndClusterState(manifest, clusterState) == true : "Manifest and ClusterState are not in sync"; - lastAcceptedManifest = manifest; + assert verifyManifestAndClusterState(manifestDetails.getClusterMetadataManifest(), clusterState) == true : "Manifest and ClusterState are not in sync"; + lastAcceptedManifest = manifestDetails.getClusterMetadataManifest(); lastAcceptedState = clusterState; + lastUploadedManifestFile = manifestDetails.getManifestFileName(); } catch (Exception e) { remoteClusterStateService.writeMetadataFailed(); handleExceptionOnWrite(e); @@ -767,12 +770,13 @@ public void markLastAcceptedStateAsCommitted() { metadataBuilder.clusterUUIDCommitted(true); clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build(); } - final ClusterMetadataManifest committedManifest = remoteClusterStateService.markLastStateAsCommitted( + final RemoteUploadDetails committedManifestDetails = remoteClusterStateService.markLastStateAsCommitted( clusterState, lastAcceptedManifest ); - lastAcceptedManifest = committedManifest; + lastAcceptedManifest = committedManifestDetails.getClusterMetadataManifest(); lastAcceptedState = clusterState; + lastUploadedManifestFile = committedManifestDetails.getManifestFileName(); } catch (Exception e) { handleExceptionOnWrite(e); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 5809ccb37b052..dbe84c6531f64 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -96,6 +96,7 @@ import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata; import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.common.remote.RemoteWritableEntityStore; +import org.opensearch.gateway.remote.model.RemoteUploadDetails; import org.opensearch.index.translog.transfer.BlobStoreTransferService; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -155,7 +156,6 @@ public class RemoteClusterStateService implements Closeable { private RemoteManifestManager remoteManifestManager; private ClusterSettings clusterSettings; private NamedWriteableRegistry namedWriteableRegistry; - private String latestManifestFileName; private final String CLUSTER_STATE_UPLOAD_TIME_LOG_STRING = "writing cluster state for version [{}] took [{}ms]"; private final String METADATA_UPDATE_LOG_STRING = "wrote metadata for [{}] indices and skipped [{}] unchanged " + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " @@ -211,7 +211,7 @@ public RemoteClusterStateService( * @return A manifest object which contains the details of uploaded entity metadata. */ @Nullable - public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, String previousClusterUUID) throws IOException { + public RemoteUploadDetails writeFullMetadata(ClusterState clusterState, String previousClusterUUID) throws IOException { logger.info("WRITING FULL STATE"); final long startTimeNanos = relativeTimeNanosSupplier.getAsLong(); if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { @@ -235,16 +235,15 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri true ); - final Tuple manifestDetails = remoteManifestManager.uploadManifest( + final RemoteUploadDetails manifestDetails = remoteManifestManager.uploadManifest( clusterState, uploadedMetadataResults, previousClusterUUID, new ClusterStateDiffManifest(clusterState, ClusterState.EMPTY_STATE), false ); - this.latestManifestFileName = manifestDetails.v2(); - logger.info("MANIFEST IN FULL STATE {}", manifestDetails.v1()); + logger.info("MANIFEST IN FULL STATE {}", manifestDetails.getClusterMetadataManifest()); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); @@ -262,7 +261,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri uploadedMetadataResults.uploadedIndexMetadata.size() ); } - return manifestDetails.v1(); + return manifestDetails; } /** @@ -273,7 +272,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri * @return The uploaded ClusterMetadataManifest file */ @Nullable - public ClusterMetadataManifest writeIncrementalMetadata( + public RemoteUploadDetails writeIncrementalMetadata( ClusterState previousClusterState, ClusterState clusterState, ClusterMetadataManifest previousManifest @@ -427,23 +426,22 @@ public ClusterMetadataManifest writeIncrementalMetadata( uploadedMetadataResults.uploadedIndexMetadata = new ArrayList<>(allUploadedIndexMetadata.values()); uploadedMetadataResults.uploadedIndicesRoutingMetadata = allUploadedIndicesRouting; - final Tuple manifestDetails = remoteManifestManager.uploadManifest( + final RemoteUploadDetails manifestDetails = remoteManifestManager.uploadManifest( clusterState, uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), new ClusterStateDiffManifest(clusterState, previousClusterState), false ); - this.latestManifestFileName = manifestDetails.v2(); - logger.info("MANIFEST IN INC STATE {}", manifestDetails.v1()); + logger.info("MANIFEST IN INC STATE {}", manifestDetails.getClusterMetadataManifest()); final long durationMillis = TimeValue.nsecToMSec(relativeTimeNanosSupplier.getAsLong() - startTimeNanos); remoteStateStats.stateSucceeded(); remoteStateStats.stateTook(durationMillis); ParameterizedMessage clusterStateUploadTimeMessage = new ParameterizedMessage( CLUSTER_STATE_UPLOAD_TIME_LOG_STRING, - manifestDetails.v1().getStateVersion(), + manifestDetails.getClusterMetadataManifest().getStateVersion(), durationMillis ); ParameterizedMessage metadataUpdateMessage = new ParameterizedMessage( @@ -475,7 +473,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( "writing cluster state for version [{}] took [{}ms]; " + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, coordination metadata updated : [{}], " + "settings metadata updated : [{}], templates metadata updated : [{}], custom metadata updated : [{}]", - manifestDetails.v1().getStateVersion(), + manifestDetails.getClusterMetadataManifest().getStateVersion(), durationMillis, numIndicesUpdated, numIndicesUnchanged, @@ -485,7 +483,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( customsToUpload.size() ); } - return manifestDetails.v1(); + return manifestDetails; } private UploadedMetadataResults writeMetadataInParallel( @@ -798,7 +796,7 @@ public RemoteManifestManager getRemoteManifestManager() { } @Nullable - public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) + public RemoteUploadDetails markLastStateAsCommitted(ClusterState clusterState, ClusterMetadataManifest previousManifest) throws IOException { assert clusterState != null : "Last accepted cluster state is not set"; if (clusterState.nodes().isLocalNodeElectedClusterManager() == false) { @@ -820,23 +818,18 @@ public ClusterMetadataManifest markLastStateAsCommitted(ClusterState clusterStat previousManifest.getClusterStateCustomMap() ); - Tuple committedManifestDetails = remoteManifestManager.uploadManifest( + RemoteUploadDetails committedManifestDetails = remoteManifestManager.uploadManifest( clusterState, uploadedMetadataResults, previousManifest.getPreviousClusterUUID(), previousManifest.getDiffManifest(), true ); - this.latestManifestFileName = committedManifestDetails.v2(); - if (!previousManifest.isClusterUUIDCommitted() && committedManifestDetails.v1().isClusterUUIDCommitted()) { - remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifestDetails.v1()); + if (!previousManifest.isClusterUUIDCommitted() && committedManifestDetails.getClusterMetadataManifest().isClusterUUIDCommitted()) { + remoteClusterStateCleanupManager.deleteStaleClusterUUIDs(clusterState, committedManifestDetails.getClusterMetadataManifest()); } - return committedManifestDetails.v1(); - } - - public String getLastUploadedManifestFileName() { - return this.latestManifestFileName; + return committedManifestDetails; } /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 7deb76fd8c0a0..1379de54f26bf 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -39,6 +39,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest; +import org.opensearch.gateway.remote.model.RemoteUploadDetails; import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -72,7 +73,7 @@ public class RemoteManifestManager { this.blobStoreRepository = blobStoreRepository; } - Tuple uploadManifest( + RemoteUploadDetails uploadManifest( ClusterState clusterState, RemoteClusterStateUtils.UploadedMetadataResults uploadedMetadataResult, String previousClusterUUID, @@ -107,7 +108,7 @@ Tuple uploadManifest( .hashesOfConsistentSettings(uploadedMetadataResult.uploadedHashesOfConsistentSettings); final ClusterMetadataManifest manifest = manifestBuilder.build(); String manifestFileName = writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest); - return new Tuple<>(manifest, manifestFileName); + return new RemoteUploadDetails(manifest, manifestFileName); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteUploadDetails.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteUploadDetails.java new file mode 100644 index 0000000000000..100280c454e43 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteUploadDetails.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.gateway.remote.ClusterMetadataManifest; + +public class RemoteUploadDetails { + + private final ClusterMetadataManifest clusterMetadataManifest; + private final String manifestFileName; + + public RemoteUploadDetails(final ClusterMetadataManifest manifest, final String manifestFileName) { + this.clusterMetadataManifest = manifest; + this.manifestFileName = manifestFileName; + } + + public ClusterMetadataManifest getClusterMetadataManifest() { + return clusterMetadataManifest; + } + + public String getManifestFileName() { + return manifestFileName; + } +} diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java index bd71aecf89101..4ade19b3d7219 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinationStateTests.java @@ -48,6 +48,7 @@ import org.opensearch.gateway.GatewayMetaState.RemotePersistedState; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; +import org.opensearch.gateway.remote.model.RemoteUploadDetails; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.EqualsHashCodeTestUtils; import org.opensearch.test.OpenSearchTestCase; @@ -944,7 +945,8 @@ public void testHandlePrePublishAndCommitWhenRemoteStateEnabled() throws IOExcep .previousClusterUUID(randomAlphaOfLength(10)) .clusterUUIDCommitted(true) .build(); - Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)).thenReturn(manifest); + Mockito.when(remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID)) + .thenReturn(new RemoteUploadDetails(manifest, "path/to/manifest")); final PersistedStateRegistry persistedStateRegistry = persistedStateRegistry(); persistedStateRegistry.addPersistedState(PersistedStateType.LOCAL, ps1); diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 418e6d8de6adb..1d63caafbb4dd 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -69,6 +69,7 @@ import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteClusterStateService; import org.opensearch.gateway.remote.RemotePersistenceStats; +import org.opensearch.gateway.remote.model.RemoteUploadDetails; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -723,9 +724,10 @@ public void testRemotePersistedState() throws IOException { final RemoteClusterStateService remoteClusterStateService = Mockito.mock(RemoteClusterStateService.class); final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder().clusterTerm(1L).stateVersion(5L).build(); final String previousClusterUUID = "prev-cluster-uuid"; - Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())).thenReturn(manifest); + Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())).thenReturn(new RemoteUploadDetails(manifest, "path/to/manifest")); - Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(manifest); + Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(new RemoteUploadDetails(manifest, "path/to/manifest")); CoordinationState.PersistedState remotePersistedState = new RemotePersistedState(remoteClusterStateService, previousClusterUUID); assertThat(remotePersistedState.getLastAcceptedState(), nullValue()); @@ -779,9 +781,10 @@ public void testRemotePersistedStateNotCommitted() throws IOException { .build(); Mockito.when(remoteClusterStateService.getLatestClusterMetadataManifest(Mockito.any(), Mockito.any())) .thenReturn(Optional.of(manifest)); - Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())).thenReturn(manifest); + Mockito.when(remoteClusterStateService.writeFullMetadata(Mockito.any(), Mockito.any())).thenReturn(new RemoteUploadDetails(manifest, "path/to/manifest")); - Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(manifest); + Mockito.when(remoteClusterStateService.writeIncrementalMetadata(Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(new RemoteUploadDetails(manifest, "path/to/manifest")); CoordinationState.PersistedState remotePersistedState = new RemotePersistedState( remoteClusterStateService, ClusterState.UNKNOWN_UUID diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 696906d25fdbb..29e581ecbd599 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -52,6 +52,7 @@ import org.opensearch.gateway.remote.model.RemoteIndexMetadata; import org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata; import org.opensearch.gateway.remote.model.RemoteTemplatesMetadata; +import org.opensearch.gateway.remote.model.RemoteUploadDetails; import org.opensearch.index.remote.RemoteIndexPathUploader; import org.opensearch.indices.IndicesModule; import org.opensearch.repositories.FilterRepository; @@ -191,7 +192,8 @@ public void teardown() throws Exception { public void testFailWriteFullMetadataNonClusterManagerNode() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().build(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, randomAlphaOfLength(10)) + .getClusterMetadataManifest(); Assert.assertThat(manifest, nullValue()); } @@ -227,7 +229,8 @@ public void testWriteFullMetadataSuccess() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); List indices = List.of(uploadedIndexMetadata); @@ -271,7 +274,8 @@ public void testWriteFullMetadataInParallelSuccess() throws IOException { }).when(container).asyncBlobUpload(writeContextArgumentCaptor.capture(), actionListenerArgumentCaptor.capture()); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); List indices = List.of(uploadedIndexMetadata); @@ -410,7 +414,8 @@ public void testWriteFullMetadataInParallelFailureForIndexMetadata() throws IOEx public void testFailWriteIncrementalMetadataNonClusterManagerNode() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().build(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata(clusterState, clusterState, null); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeIncrementalMetadata(clusterState, clusterState, null) + .getClusterMetadataManifest(); Assert.assertThat(manifest, nullValue()); assertEquals(0, remoteClusterStateService.getStats().getSuccessCount()); } @@ -442,7 +447,7 @@ public void testWriteIncrementalMetadataSuccess() throws IOException { previousClusterState, clusterState, previousManifest - ); + ).getClusterMetadataManifest(); final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename"); final List indices = List.of(uploadedIndexMetadata); @@ -517,7 +522,7 @@ private void verifyCodecMigrationManifest(int previousCodec) throws IOException previousClusterState, newClusterState, previousManifest - ); + ).getClusterMetadataManifest(); // global metadata is updated assertThat(manifestAfterUpdate.hasMetadataAttributesFiles(), is(true)); @@ -554,7 +559,7 @@ private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMe previousClusterState, clusterState, previousManifest - ); + ).getClusterMetadataManifest(); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .codecVersion(2) @@ -745,7 +750,8 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException { // Initial cluster state with index. final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_"); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_") + .getClusterMetadataManifest(); ClusterState clusterState1 = ClusterState.builder(initialClusterState) .metadata( @@ -760,7 +766,7 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException { initialClusterState, clusterState1, initialManifest - ); + ).getClusterMetadataManifest(); // remove custom1 from the cluster state, update custom2, custom3 is at it is, added custom4 ClusterState clusterState2 = ClusterState.builder(initialClusterState) .metadata( @@ -770,7 +776,8 @@ public void testCustomMetadataDeletedUpdatedAndAdded() throws IOException { .putCustom("custom4", new CustomMetadata1("mock_custom_metadata4")) ) .build(); - ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1); + ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1) + .getClusterMetadataManifest(); // custom1 is removed assertFalse(manifest2.getCustomMetadataMap().containsKey("custom1")); // custom2 is updated @@ -820,7 +827,8 @@ public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException { // Initial cluster state with index. final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_"); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_") + .getClusterMetadataManifest(); String initialIndex = "test-index"; Index index1 = new Index("test-index-1", "index-uuid-1"); Index index2 = new Index("test-index-2", "index-uuid-2"); @@ -853,7 +861,7 @@ public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException { initialClusterState, clusterState1, initialManifest - ); + ).getClusterMetadataManifest(); // verify that initial index is removed, and new index are added assertEquals(1, initialManifest.getIndices().size()); assertEquals(2, manifest1.getIndices().size()); @@ -864,7 +872,8 @@ public void testIndexMetadataDeletedUpdatedAndAdded() throws IOException { ClusterState clusterState2 = ClusterState.builder(clusterState1) .metadata(Metadata.builder(clusterState1.getMetadata()).put(indexMetadata1, true).build()) .build(); - ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1); + ClusterMetadataManifest manifest2 = remoteClusterStateService.writeIncrementalMetadata(clusterState1, clusterState2, manifest1) + .getClusterMetadataManifest(); // index1 is updated assertEquals(2, manifest2.getIndices().size()); assertEquals( @@ -897,7 +906,8 @@ private void verifyMetadataAttributeOnlyUpdated( // Initial cluster state with index. final ClusterState initialClusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); remoteClusterStateService.start(); - final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_"); + final ClusterMetadataManifest initialManifest = remoteClusterStateService.writeFullMetadata(initialClusterState, "_na_") + .getClusterMetadataManifest(); ClusterState newClusterState = clusterStateUpdater.apply(initialClusterState); @@ -908,9 +918,10 @@ private void verifyMetadataAttributeOnlyUpdated( initialClusterState, newClusterState, initialManifest - ); + ).getClusterMetadataManifest(); } else { - manifestAfterMetadataUpdate = remoteClusterStateService.writeFullMetadata(newClusterState, initialClusterState.stateUUID()); + manifestAfterMetadataUpdate = remoteClusterStateService.writeFullMetadata(newClusterState, initialClusterState.stateUUID()) + .getClusterMetadataManifest(); } assertions.accept(initialManifest, manifestAfterMetadataUpdate); @@ -1280,7 +1291,8 @@ public void testRemoteStateStats() throws IOException { final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); mockBlobStoreObjects(); remoteClusterStateService.start(); - final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid"); + final ClusterMetadataManifest manifest = remoteClusterStateService.writeFullMetadata(clusterState, "prev-cluster-uuid") + .getClusterMetadataManifest(); assertTrue(remoteClusterStateService.getStats() != null); assertEquals(1, remoteClusterStateService.getStats().getSuccessCount()); @@ -1690,6 +1702,10 @@ private String getFileNameFromPath(String filePath) { return splitPath[splitPath.length - 1]; } + private static RemoteUploadDetails getRemoteUploadDetails(ClusterMetadataManifest manifest) { + return new RemoteUploadDetails(manifest, "path/to/manifest"); + } + static ClusterState.Builder generateClusterStateWithGlobalMetadata() { final Settings clusterSettings = Settings.builder().put("cluster.blocks.read_only", true).build(); final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); From fa249885fcc59d26b8e8bccdb0736f7c7d61e487 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Fri, 7 Jun 2024 20:22:52 +0530 Subject: [PATCH 129/133] Send manifest file name in remote publish request Signed-off-by: Sooraj Sinha --- .../opensearch/cluster/coordination/Coordinator.java | 2 +- .../coordination/PublicationTransportHandler.java | 11 ++++++----- .../cluster/coordination/RemotePublishRequest.java | 1 - .../java/org/opensearch/gateway/GatewayMetaState.java | 5 +++++ .../PublicationTransportHandlerTests.java | 2 +- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 944f586dc3ff2..4a258a92d0ca6 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -1327,7 +1327,7 @@ assert getLocalNode().equals(clusterState.getNodes().get(getLocalNode().getId()) final PublicationTransportHandler.PublicationContext publicationContext = publicationHandler.newPublicationContext( clusterChangedEvent, coordinationState.get().isRemotePublicationEnabled(), - ((RemotePersistedState) persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE)) + persistedStateRegistry ); final PublishRequest publishRequest = coordinationState.get().handleClientValue(clusterState); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java index fb4893e46c345..545667ce75556 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/PublicationTransportHandler.java @@ -300,8 +300,8 @@ private PublishWithJoinResponse acceptStateOnLocalNode(RemotePublishRequest remo } public PublicationContext newPublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled, - String manifestFileName) { - final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemotePublicationEnabled, manifestFileName); + PersistedStateRegistry persistedStateRegistry) { + final PublicationContext publicationContext = new PublicationContext(clusterChangedEvent, isRemotePublicationEnabled, persistedStateRegistry); // Build the serializations we expect to need now, early in the process, so that an error during serialization fails the publication // straight away. This isn't watertight since we send diffs on a best-effort basis and may fall back to sending a full state (and @@ -347,15 +347,15 @@ public class PublicationContext { private final Map serializedStates = new HashMap<>(); private final Map serializedDiffs = new HashMap<>(); private final boolean sendRemoteState; - private final String manifestFileName; + private final PersistedStateRegistry persistedStateRegistry; - PublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled, String manifestFileName) { + PublicationContext(ClusterChangedEvent clusterChangedEvent, boolean isRemotePublicationEnabled, PersistedStateRegistry persistedStateRegistry) { discoveryNodes = clusterChangedEvent.state().nodes(); newState = clusterChangedEvent.state(); previousState = clusterChangedEvent.previousState(); sendFullVersion = previousState.getBlocks().disableStatePersistence(); sendRemoteState = isRemotePublicationEnabled; - this.manifestFileName = manifestFileName; + this.persistedStateRegistry = persistedStateRegistry; } void buildDiffAndSerializeStates() { @@ -468,6 +468,7 @@ public String executor() { private void sendRemoteClusterState(final DiscoveryNode destination, final ClusterState clusterState, final ActionListener listener) { try { + final String manifestFileName = ((RemotePersistedState) persistedStateRegistry.getPersistedState(PersistedStateType.REMOTE)).getLastUploadedManifestFile(); final RemotePublishRequest remotePublishRequest = new RemotePublishRequest(discoveryNodes.getLocalNode(), clusterState.term(), clusterState.getVersion(), clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifestFileName); final Consumer transportExceptionHandler = exp -> { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java index abbe1f0f78421..68dea53372c45 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/RemotePublishRequest.java @@ -16,7 +16,6 @@ public class RemotePublishRequest extends TermVersionRequest { - // todo Do we need cluster name and UUID ? private final String clusterName; private final String clusterUUID; private final String manifestFile; diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java index ffa0c8392876b..1cc0c9cbb6642 100644 --- a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java @@ -666,6 +666,7 @@ public static class RemotePersistedState implements PersistedState { private ClusterState lastAcceptedState; private ClusterMetadataManifest lastAcceptedManifest; + private String lastUploadedManifestFile; private final RemoteClusterStateService remoteClusterStateService; private String previousClusterUUID; @@ -692,6 +693,10 @@ public void setCurrentTerm(long currentTerm) { // But for RemotePersistedState, the state is only pushed by the active cluster. So this method is not required. } + public String getLastUploadedManifestFile() { + return lastUploadedManifestFile; + } + @Override public void setLastAcceptedState(ClusterState clusterState) { try { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java index 27ebbf77fa28e..a9950d82bfac8 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/PublicationTransportHandlerTests.java @@ -112,7 +112,7 @@ public void writeTo(StreamOutput out) throws IOException { OpenSearchException e = expectThrows( OpenSearchException.class, - () -> handler.newPublicationContext(new ClusterChangedEvent("test", unserializableClusterState, clusterState), false) + () -> handler.newPublicationContext(new ClusterChangedEvent("test", unserializableClusterState, clusterState), false, null) ); assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IOException.class)); From 5c6f418316afeb5afc5e23301a338d207db04f1d Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sat, 8 Jun 2024 22:30:17 +0530 Subject: [PATCH 130/133] Create serde utility for Writable classes Signed-off-by: Sooraj Sinha --- .../remote/model/RemoteClusterBlocks.java | 16 +-- .../ChecksumWritableBlobStoreFormat.java | 117 ++++++++++++++++++ 2 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index 707c4d54e04fa..34741da435b44 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -8,28 +8,22 @@ package org.opensearch.gateway.remote.model; -import static org.opensearch.core.common.bytes.BytesReference.toBytes; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; import java.io.IOException; import java.io.InputStream; import java.util.List; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.common.io.Streams; -import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; -import org.opensearch.core.common.io.stream.BytesStreamInput; -import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; -import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import org.opensearch.repositories.blobstore.ChecksumWritableBlobStoreFormat; /** * Wrapper class for uploading/downloading {@link ClusterBlocks} to/from remote blob store @@ -37,6 +31,7 @@ public class RemoteClusterBlocks extends AbstractRemoteWritableBlobEntity { public static final String CLUSTER_BLOCKS = "blocks"; + public static final ChecksumWritableBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumWritableBlobStoreFormat<>("blocks", ClusterBlocks::readFrom); private ClusterBlocks clusterBlocks; private long stateVersion; @@ -80,14 +75,11 @@ public UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - clusterBlocks.writeTo(bytesStreamOutput); - return bytesStreamOutput.bytes().streamInput(); + return CLUSTER_BLOCKS_FORMAT.serialize(clusterBlocks, generateBlobFileName(), getCompressor()).streamInput(); } @Override public ClusterBlocks deserialize(final InputStream inputStream) throws IOException { - StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream))); - return ClusterBlocks.readFrom(in); + return CLUSTER_BLOCKS_FORMAT.deserialize(blobName, ClusterBlocks::readFrom, Streams.readFully(inputStream)); } } diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java new file mode 100644 index 0000000000000..96dee617bd082 --- /dev/null +++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormat.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.blobstore; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Objects; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.store.ByteBuffersDataInput; +import org.apache.lucene.store.ByteBuffersIndexInput; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.OutputStreamIndexOutput; +import org.apache.lucene.util.BytesRef; +import org.opensearch.Version; +import org.opensearch.common.CheckedFunction; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; +import org.opensearch.common.lucene.store.IndexOutputOutputStream; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.CompressorRegistry; +import org.opensearch.core.compress.NotXContentException; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.gateway.CorruptStateException; + +public class ChecksumWritableBlobStoreFormat { + + public static final int VERSION = 1; + + private static final int BUFFER_SIZE = 4096; + + private final String codec; + private final CheckedFunction reader; + + public ChecksumWritableBlobStoreFormat(String codec, CheckedFunction reader) { + this.codec = codec; + this.reader = reader; + } + + public BytesReference serialize( + final T obj, + final String blobName, + final Compressor compressor + ) throws IOException { + try (BytesStreamOutput outputStream = new BytesStreamOutput()) { + try ( + OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput( + "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")", + blobName, + outputStream, + BUFFER_SIZE + ) + ) { + CodecUtil.writeHeader(indexOutput, codec, VERSION); + + try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) { + @Override + public void close() throws IOException { + // this is important since some of the XContentBuilders write bytes on close. + // in order to write the footer we need to prevent closing the actual index input. + } + }; + StreamOutput stream = new OutputStreamStreamOutput(compressor.threadLocalOutputStream(indexOutputOutputStream)); + ) { + // TODO The stream version should be configurable + stream.setVersion(Version.CURRENT); + obj.writeTo(stream); + } + CodecUtil.writeFooter(indexOutput); + } + return outputStream.bytes(); + } + } + + public T deserialize(String blobName, CheckedFunction reader, BytesReference bytes) throws IOException { + final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; + try { + final IndexInput indexInput = bytes.length() > 0 + ? new ByteBuffersIndexInput(new ByteBuffersDataInput(Arrays.asList(BytesReference.toByteBuffers(bytes))), resourceDesc) + : new ByteArrayIndexInput(resourceDesc, BytesRef.EMPTY_BYTES); + CodecUtil.checksumEntireFile(indexInput); + CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION); + long filePointer = indexInput.getFilePointer(); + long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; + BytesReference bytesReference = bytes.slice((int) filePointer, (int) contentSize); + Compressor compressor; + try { + compressor = CompressorRegistry.compressor(bytesReference); + } catch (NotXContentException e) { + compressor = CompressorRegistry.none(); + } + try (StreamInput in = new InputStreamStreamInput(compressor.threadLocalInputStream(bytesReference.streamInput()))) { + return reader.apply(in); + } + } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { + // we trick this into a dedicated exception with the original stacktrace + throw new CorruptStateException(ex); + } + } + +} From a813834b6842989eb68752c4675a2260113db0e6 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sat, 8 Jun 2024 22:44:02 +0530 Subject: [PATCH 131/133] ChecksumWritableBlobStoreFormat Unit test Signed-off-by: Sooraj Sinha --- .../ChecksumWritableBlobStoreFormatTests.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java new file mode 100644 index 0000000000000..68dda7bdc6dde --- /dev/null +++ b/server/src/test/java/org/opensearch/repositories/blobstore/ChecksumWritableBlobStoreFormatTests.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.blobstore; + +import static org.hamcrest.Matchers.is; + +import java.io.IOException; +import org.junit.Before; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.compress.CompressorRegistry; +import org.opensearch.core.index.Index; +import org.opensearch.test.OpenSearchTestCase; + +public class ChecksumWritableBlobStoreFormatTests extends OpenSearchTestCase { + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long VERSION = 5L; + + private final ChecksumWritableBlobStoreFormat clusterBlocksFormat = new ChecksumWritableBlobStoreFormat<>("index-metadata", IndexMetadata::readFrom); + + public void testSerDe() throws IOException { + IndexMetadata indexMetadata = getIndexMetadata(); + BytesReference bytesReference = clusterBlocksFormat.serialize(indexMetadata, TEST_BLOB_FILE_NAME, CompressorRegistry.none()); + IndexMetadata readIndexMetadata = clusterBlocksFormat.deserialize(TEST_BLOB_FILE_NAME, IndexMetadata::readFrom, bytesReference); + assertThat(readIndexMetadata, is(indexMetadata)); + } + + private IndexMetadata getIndexMetadata() { + final Index index = new Index("test-index", "index-uuid"); + final Settings idxSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); + return new IndexMetadata.Builder(index.getName()).settings(idxSettings) + .version(VERSION) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + } +} From e8c8413d0862e25c63e5348cb9abbc52c9ad6ea2 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha Date: Sat, 8 Jun 2024 22:44:19 +0530 Subject: [PATCH 132/133] Fixing build Signed-off-by: Sooraj Sinha --- .../gateway/GatewayMetaStatePersistedStateTests.java | 3 ++- .../remote/RemoteClusterStateServiceTests.java | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java index 1d63caafbb4dd..5ee819618daa4 100644 --- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -489,7 +489,8 @@ public void testDataOnlyNodePersistence() throws Exception { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) + List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + null ); } else { return null; diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 29e581ecbd599..0e2c3efae428d 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -179,7 +179,8 @@ public void setup() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) + List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + null ); } @@ -209,7 +210,8 @@ public void testFailInitializationWhenRemoteStateDisabled() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) + List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + null ) ); } @@ -1186,7 +1188,8 @@ public void testMarkLastStateAsCommittedSuccess() throws IOException { List indices = List.of(uploadedIndexMetadata); final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(indices).build(); - final ClusterMetadataManifest manifest = remoteClusterStateService.markLastStateAsCommitted(clusterState, previousManifest); + final RemoteUploadDetails manifestDetails = remoteClusterStateService.markLastStateAsCommitted(clusterState, previousManifest); + ClusterMetadataManifest manifest = manifestDetails.getClusterMetadataManifest(); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() .indices(indices) @@ -1323,7 +1326,8 @@ public void testRemoteRoutingTableInitializedWhenEnabled() { clusterService, () -> 0L, threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)) + List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), + null ); assertNotNull(remoteClusterStateService.getRemoteRoutingTableService()); } From 4def30f12ad4567c022b87abc72471affb57d994 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 17:28:38 +0530 Subject: [PATCH 133/133] Handle read ClusterState for V1 manifest Signed-off-by: Shivansh Arora --- .../remote/RemoteClusterStateService.java | 67 ++++++--- .../remote/RemoteGlobalMetadataManager.java | 133 ++++++------------ 2 files changed, 94 insertions(+), 106 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index dbe84c6531f64..d7461f9f06475 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -8,7 +8,11 @@ package org.opensearch.gateway.remote; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTE; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; @@ -222,7 +226,7 @@ public RemoteUploadDetails writeFullMetadata(ClusterState clusterState, String p UploadedMetadataResults uploadedMetadataResults = writeMetadataInParallel( clusterState, new ArrayList<>(clusterState.metadata().indices().values()), - Collections.emptyMap(), + emptyMap(), clusterState.metadata().customs(), true, true, @@ -1226,23 +1230,48 @@ private ClusterState readClusterStateInParallel( public ClusterState getClusterStateForManifest(String clusterName, ClusterMetadataManifest manifest, String localNodeId, boolean includeEphemeral) throws IOException { - return readClusterStateInParallel( - ClusterState.builder(new ClusterName(clusterName)).build(), - manifest, - manifest.getClusterUUID(), - localNodeId, - manifest.getIndices(), - manifest.getCustomMetadataMap(), - manifest.getCoordinationMetadata() != null, - manifest.getSettingsMetadata() != null, - manifest.getTransientSettingsMetadata() != null, - manifest.getTemplatesMetadata() != null, - includeEphemeral && manifest.getDiscoveryNodesMetadata() != null, - includeEphemeral && manifest.getClusterBlocksMetadata() != null, - includeEphemeral ? manifest.getIndicesRouting() : Collections.emptyList(), - includeEphemeral && manifest.getHashesOfConsistentSettings() != null, - includeEphemeral ? manifest.getClusterStateCustomMap() : Collections.emptyMap() - ); + if (manifest.onOrAfterCodecVersion(CODEC_V2)) { + return readClusterStateInParallel( + ClusterState.builder(new ClusterName(clusterName)).build(), + manifest, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + manifest.getCustomMetadataMap(), + manifest.getCoordinationMetadata() != null, + manifest.getSettingsMetadata() != null, + manifest.getTransientSettingsMetadata() != null, + manifest.getTemplatesMetadata() != null, + includeEphemeral && manifest.getDiscoveryNodesMetadata() != null, + includeEphemeral && manifest.getClusterBlocksMetadata() != null, + includeEphemeral ? manifest.getIndicesRouting() : emptyList(), + includeEphemeral && manifest.getHashesOfConsistentSettings() != null, + includeEphemeral ? manifest.getClusterStateCustomMap() : emptyMap() + ); + } else { + ClusterState clusterState = readClusterStateInParallel( + ClusterState.builder(new ClusterName(clusterName)).build(), + manifest, + manifest.getClusterUUID(), + localNodeId, + manifest.getIndices(), + // for manifest codec V1, we don't have the following objects to read, so not passing anything + emptyMap(), + false, + false, + false, + false, + false, + false, + emptyList(), + false, + emptyMap() + ); + Metadata.Builder mb = Metadata.builder(remoteGlobalMetadataManager.getGlobalMetadata(manifest.getClusterUUID(), manifest)); + mb.indices(clusterState.metadata().indices()); + return ClusterState.builder(clusterState).metadata(mb).build(); + } + } public ClusterState getClusterStateUsingDiff(String clusterName, ClusterMetadataManifest manifest, ClusterState previousState, String localNodeId) @@ -1413,7 +1442,7 @@ private List createClusterChain(final Map 1) { logger.info("Top level cluster UUIDs: {}", topLevelClusterUUIDs); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java index d1153a0a90809..731cc9094b06d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteGlobalMetadataManager.java @@ -195,31 +195,58 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe compressor, namedXContentRegistry); return globalMetadataBlobStore.read(remoteGlobalMetadata); } else if (clusterMetadataManifest.hasMetadataAttributesFiles()) { - CoordinationMetadata coordinationMetadata = getCoordinationMetadata( - clusterUUID, - clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() - ); - Settings settingsMetadata = getSettingsMetadata( - clusterUUID, - clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() - ); - TemplatesMetadata templatesMetadata = getTemplatesMetadata( - clusterUUID, - clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() - ); Metadata.Builder builder = new Metadata.Builder(); - builder.coordinationMetadata(coordinationMetadata); - builder.persistentSettings(settingsMetadata); - builder.templates(templatesMetadata); + if (clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename() != null) { + RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata( + clusterMetadataManifest.getCoordinationMetadata().getUploadedFilename(), + clusterUUID, + compressor, + namedXContentRegistry + ); + builder.coordinationMetadata(coordinationMetadataBlobStore.read(remoteCoordinationMetadata)); + } + if (clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename() != null) { + RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata( + clusterMetadataManifest.getTemplatesMetadata().getUploadedFilename(), + clusterUUID, + compressor, + namedXContentRegistry + ); + builder.templates(templatesMetadataBlobStore.read(remoteTemplatesMetadata)); + } + if (clusterMetadataManifest.getSettingsMetadata().getUploadedFilename() != null) { + RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata( + clusterMetadataManifest.getSettingsMetadata().getUploadedFilename(), + clusterUUID, + compressor, + namedXContentRegistry + ); + builder.persistentSettings(persistentSettingsBlobStore.read(remotePersistentSettingsMetadata)); + } builder.clusterUUID(clusterMetadataManifest.getClusterUUID()); builder.clusterUUIDCommitted(clusterMetadataManifest.isClusterUUIDCommitted()); builder.version(clusterMetadataManifest.getMetadataVersion()); clusterMetadataManifest.getCustomMetadataMap() .forEach( - (key, value) -> builder.putCustom( - key, - getCustomsMetadata(clusterUUID, value.getUploadedFilename(), key) - ) + (key, value) -> { + try { + builder.putCustom( + key, + customMetadataBlobStore.read(new RemoteCustomMetadata( + value.getUploadedFilename(), + key, + clusterUUID, + compressor, + namedXContentRegistry + )) + ); + } catch (IOException e) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", value.getUploadedFilename()), + e + ); + } + } ); return builder.build(); } else { @@ -233,74 +260,6 @@ Metadata getGlobalMetadata(String clusterUUID, ClusterMetadataManifest clusterMe } } - public CoordinationMetadata getCoordinationMetadata(String clusterUUID, String coordinationMetadataFileName) { - try { - // Fetch Coordination metadata - if (coordinationMetadataFileName != null) { - RemoteCoordinationMetadata remoteCoordinationMetadata = new RemoteCoordinationMetadata(coordinationMetadataFileName, clusterUUID, - compressor, namedXContentRegistry); - return coordinationMetadataBlobStore.read(remoteCoordinationMetadata); - } else { - return CoordinationMetadata.EMPTY_METADATA; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Coordination Metadata - %s", coordinationMetadataFileName), - e - ); - } - } - - public Settings getSettingsMetadata(String clusterUUID, String settingsMetadataFileName) { - try { - // Fetch Settings metadata - if (settingsMetadataFileName != null) { - RemotePersistentSettingsMetadata remotePersistentSettingsMetadata = new RemotePersistentSettingsMetadata(settingsMetadataFileName, clusterUUID, - compressor, namedXContentRegistry); - return persistentSettingsBlobStore.read(remotePersistentSettingsMetadata); - } else { - return Settings.EMPTY; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Settings Metadata - %s", settingsMetadataFileName), - e - ); - } - } - - public TemplatesMetadata getTemplatesMetadata(String clusterUUID, String templatesMetadataFileName) { - try { - // Fetch Templates metadata - if (templatesMetadataFileName != null) { - RemoteTemplatesMetadata remoteTemplatesMetadata = new RemoteTemplatesMetadata(templatesMetadataFileName, clusterUUID, - compressor, namedXContentRegistry); - return templatesMetadataBlobStore.read(remoteTemplatesMetadata); - } else { - return TemplatesMetadata.EMPTY_METADATA; - } - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Templates Metadata - %s", templatesMetadataFileName), - e - ); - } - } - - public Metadata.Custom getCustomsMetadata(String clusterUUID, String customMetadataFileName, String custom) { - requireNonNull(customMetadataFileName); - try { - // Fetch Custom metadata - RemoteCustomMetadata remoteCustomMetadata = new RemoteCustomMetadata(customMetadataFileName, custom, clusterUUID, compressor, namedXContentRegistry); - return customMetadataBlobStore.read(remoteCustomMetadata); - } catch (IOException e) { - throw new IllegalStateException( - String.format(Locale.ROOT, "Error while downloading Custom Metadata - %s", customMetadataFileName), - e - ); - } - } - Map getUpdatedCustoms(ClusterState currentState, ClusterState previousState) { if (Metadata.isCustomMetadataEqual(previousState.metadata(), currentState.metadata())) { return new HashMap<>();