Skip to content

Commit

Permalink
Address PR comments
Browse files Browse the repository at this point in the history 
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
@shiv0408
shiv0408 committed Mar 15, 2024
1 parent fc270d1 commit adb4cf2
Show file tree
Hide file tree
Showing 5 changed files with 84 additions and 61 deletions.
33 changes: 16 additions & 17 deletions ...c/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,24 @@

package org.opensearch.gateway.remote;

import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.blobstore.BlobPath;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
import org.opensearch.discovery.DiscoveryStats;
import org.opensearch.monitor.fs.FsInfo;
import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
import org.opensearch.repositories.RepositoriesService;
import org.opensearch.repositories.blobstore.BlobStoreRepository;
import org.opensearch.test.OpenSearchIntegTestCase;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA;
import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA;
Expand Down Expand Up @@ -198,21 +189,29 @@ public void testRemoteStateStatsFromAllNodes() {
}
}

public void testRemoteRestoreOnClusterManagerRestartCorruptedLocal() throws IOException {
public void testRemoteClusterStateMetadataSplit() throws IOException {
initialTestSetup(1, 0, 1, 1);

RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance(
RemoteClusterStateService.class
);
RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class);
BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME);
BlobPath globalMetadataPath = repository.basePath().add(
Base64.getUrlEncoder().withoutPadding()
.encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8))
).add("cluster-state").add(getClusterState().metadata().clusterUUID()).add("global-metadata");

Map<String, Integer> metadataFiles = repository.blobStore().blobContainer(globalMetadataPath).listBlobs()
.keySet().stream()
BlobPath globalMetadataPath = repository.basePath()
.add(
Base64.getUrlEncoder()
.withoutPadding()
.encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8))
)
.add("cluster-state")
.add(getClusterState().metadata().clusterUUID())
.add("global-metadata");

Map<String, Integer> metadataFiles = repository.blobStore()
.blobContainer(globalMetadataPath)
.listBlobs()
.keySet()
.stream()
.map(fileName -> {
logger.info(fileName);
return fileName.split(DELIMITER)[0];
Expand Down
8 changes: 1 addition & 7 deletions server/src/main/java/org/opensearch/cluster/metadata/Metadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,7 @@ static Custom fromXContent(XContentParser parser, String name) throws IOExceptio

static Custom fromXContent(XContentParser parser) throws IOException {
String currentFieldName = parser.currentName();
try {
return parser.namedObject(Custom.class, currentFieldName, null);
} catch (NamedObjectNotFoundException e) {
logger.warn("Unknown custom object with type {}", currentFieldName);
parser.skipChildren();
throw e;
}
return fromXContent(parser, currentFieldName);
}
}

Expand Down
11 changes: 4 additions & 7 deletions server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@ public Builder(Map<String, IndexTemplateMetadata> templates) {
this.templates = templates;
}

public Builder put(IndexTemplateMetadata.Builder template) {
return put(template.build());
public Builder put(IndexTemplateMetadata.Builder templateBuilder) {
return put(templateBuilder.build());
}

public Builder put(IndexTemplateMetadata template) {
Expand All @@ -117,18 +117,15 @@ public TemplatesMetadata build() {
return new TemplatesMetadata(templates);
}

public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException {
// builder.startObject("templates-metadata");
for (IndexTemplateMetadata cursor : templates.getTemplates().values()) {
public static void toXContent(TemplatesMetadata templatesMetadata, XContentBuilder builder, Params params) throws IOException {
for (IndexTemplateMetadata cursor : templatesMetadata.getTemplates().values()) {
IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
}
// builder.endObject();
}

public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
Builder builder = new Builder();

// we might get here after the templates-metadata element, or on a fresh parser
XContentParser.Token token = parser.currentToken();
String currentFieldName = parser.currentName();
if (currentFieldName == null) {
Expand Down
50 changes: 23 additions & 27 deletions server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,22 +525,16 @@ private UploadedMetadataResults writeMetadataInParallel(
)
);
}
customToUpload.forEach(
(key, value) -> {
String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key);
uploadTasks.put(
customComponent,
getAsyncMetadataWriteAction(
clusterState,
customComponent,
CUSTOM_METADATA_FORMAT,
value,
listener
)
);
}
);
indexToUpload.forEach(indexMetadata -> { uploadTasks.put(indexMetadata.getIndexName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener)); });
customToUpload.forEach((key, value) -> {
String customComponent = String.join(CUSTOM_DELIMITER, CUSTOM_METADATA, key);
uploadTasks.put(
customComponent,
getAsyncMetadataWriteAction(clusterState, customComponent, CUSTOM_METADATA_FORMAT, value, listener)
);
});
indexToUpload.forEach(indexMetadata -> {
uploadTasks.put(indexMetadata.getIndexName(), getIndexMetadataAsyncAction(clusterState, indexMetadata, listener));
});

// start async upload of all required metadata files
for (CheckedRunnable<IOException> uploadTask : uploadTasks.values()) {
Expand All @@ -554,7 +548,7 @@ private UploadedMetadataResults writeMetadataInParallel(
String.format(
Locale.ROOT,
"Timed out waiting for transfer of following metadata to complete - %s",
String.join(", ", uploadTasks.keySet())
String.join(", ", uploadTasks.keySet())
)
);
exceptionList.forEach(ex::addSuppressed);
Expand Down Expand Up @@ -591,8 +585,10 @@ private UploadedMetadataResults writeMetadataInParallel(
} else if (uploadedMetadata.getComponent().contains(CUSTOM_METADATA)) {
// component name for custom metadata will look like custom--<metadata-attribute>
String custom = name.split(DELIMITER)[0].split(CUSTOM_DELIMITER)[1];
response.uploadedCustomMetadataMap.put(custom,
new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename()));
response.uploadedCustomMetadataMap.put(
custom,
new UploadedMetadataAttribute(custom, uploadedMetadata.getUploadedFilename())
);
} else if (COORDINATION_METADATA.equals(uploadedMetadata.getComponent())) {
response.uploadedCoordinationMetadata = (UploadedMetadataAttribute) uploadedMetadata;
} else if (SETTING_METADATA.equals(uploadedMetadata.getComponent())) {
Expand Down Expand Up @@ -1091,8 +1087,13 @@ private Metadata getGlobalMetadata(String clusterName, String clusterUUID, Clust
builder.coordinationMetadata(coordinationMetadata);
builder.persistentSettings(settingsMetadata);
builder.templates(templatesMetadata);
clusterMetadataManifest.getCustomMetadataMap().forEach((key, value) ->
builder.putCustom(key, getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key)));
clusterMetadataManifest.getCustomMetadataMap()
.forEach(
(key, value) -> builder.putCustom(
key,
getCustomsMetadata(clusterName, clusterUUID, value.getUploadedFilename(), key)
)
);
return builder.build();
} else {
return Metadata.EMPTY_METADATA;
Expand Down Expand Up @@ -1168,12 +1169,7 @@ private TemplatesMetadata getTemplatesMetadata(String clusterName, String cluste
}
}

private Metadata.Custom getCustomsMetadata(
String clusterName,
String clusterUUID,
String customMetadataFileName,
String custom
) {
private Metadata.Custom getCustomsMetadata(String clusterName, String clusterUUID, String customMetadataFileName, String custom) {
requireNonNull(customMetadataFileName);
try {
// Fetch Custom metadata
Expand Down
43 changes: 40 additions & 3 deletions server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
package org.opensearch.gateway.remote;

import org.opensearch.Version;
import org.opensearch.cluster.metadata.IndexGraveyard;
import org.opensearch.cluster.metadata.RepositoriesMetadata;
import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
Expand All @@ -22,9 +25,11 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;

public class ClusterMetadataManifestTests extends OpenSearchTestCase {

Expand Down Expand Up @@ -100,7 +105,23 @@ public void testClusterMetadataManifestXContent() throws IOException {
new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"),
new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"),
new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"),
new HashMap<>()
Collections.unmodifiableList(
Arrays.asList(
new UploadedMetadataAttribute(
RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE,
"custom--repositories-file"
),
new UploadedMetadataAttribute(
RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE,
"custom--index_graveyard-file"
),
new UploadedMetadataAttribute(
RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER
+ WeightedRoutingMetadata.TYPE,
"custom--weighted_routing_netadata-file"
)
)
).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()))
);
final XContentBuilder builder = JsonXContent.contentBuilder();
builder.startObject();
Expand Down Expand Up @@ -130,7 +151,23 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() {
new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"),
new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"),
new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"),
new HashMap<>()
Collections.unmodifiableList(
Arrays.asList(
new UploadedMetadataAttribute(
RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE,
"custom--repositories-file"
),
new UploadedMetadataAttribute(
RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE,
"custom--index_graveyard-file"
),
new UploadedMetadataAttribute(
RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER
+ WeightedRoutingMetadata.TYPE,
"custom--weighted_routing_netadata-file"
)
)
).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity()))
);
{ // Mutate Cluster Term
EqualsHashCodeTestUtils.checkEqualsAndHashCode(
Expand Down

0 comments on commit adb4cf2

Please sign in to comment.