From 004138604a0539369091c6a6cf98fe845afb7561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sat, 26 Aug 2023 14:20:01 +0530 Subject: [PATCH 01/34] [Remote Store] Changes to perform repository registration during bootstrap via node attributes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- CHANGELOG.md | 1 + .../cluster/remotestore/RemoteStoreNode.java | 124 +++++++++++ .../remotestore/RemoteStoreService.java | 204 ++++++++++++++++++ .../cluster/coordination/Coordinator.java | 11 +- .../cluster/coordination/JoinHelper.java | 12 +- .../coordination/JoinTaskExecutor.java | 87 +++++++- .../cluster/node/DiscoveryNode.java | 10 + .../common/settings/ClusterSettings.java | 2 + .../opensearch/discovery/DiscoveryModule.java | 7 +- .../main/java/org/opensearch/node/Node.java | 6 +- .../repositories/RepositoriesService.java | 9 +- .../indices/cluster/ClusterStateChanges.java | 18 +- 12 files changed, 478 insertions(+), 13 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d282bd2e6348..b02470a218078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add average concurrency metric for concurrent segment search ([#9670](https://github.com/opensearch-project/OpenSearch/issues/9670)) - [Remote state] Integrate remote cluster state in publish/commit flow ([#9665](https://github.com/opensearch-project/OpenSearch/pull/9665)) - [Segment Replication] Adding segment replication statistics rolled up at index, node and cluster level ([#9709](https://github.com/opensearch-project/OpenSearch/pull/9709)) +- [Remote Store] Changes to introduce repository registration during bootstrap via node attributes. ([#9105](https://github.com/opensearch-project/OpenSearch/pull/9105)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java new file mode 100644 index 0000000000000..2006d46edc443 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java @@ -0,0 +1,124 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.remotestore; + +import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.settings.Settings; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * This is an extension of {@Code DiscoveryNode} which provides an abstraction for validating and storing information + * specific to remote backed storage nodes. + * + * @opensearch.internal + */ +public class RemoteStoreNode extends DiscoveryNode { + + public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; + public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; + public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; + public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; + public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; + private final DiscoveryNode node; + private final RepositoriesMetadata repositoriesMetadata; + + /** + * Creates a new {@link RemoteStoreNode} + */ + public RemoteStoreNode(DiscoveryNode node) { + super(node.getName(), node.getId(), node.getAddress(), node.getAttributes(), node.getRoles(), node.getVersion()); + this.node = node; + this.repositoriesMetadata = buildRepositoriesMetadata(node); + } + + private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) { + String attributeValue = node.getAttributes().get(attributeKey); + if (attributeValue == null || attributeValue.isEmpty()) { + throw new IllegalStateException("joining node [" + node + "] doesn't have the node attribute [" + attributeKey + "]."); + } + + return attributeValue; + } + + private Map validateSettingsAttributesNonNull(DiscoveryNode node, String settingsAttributeKeyPrefix) { + return node.getAttributes() + .keySet() + .stream() + .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(node, key))); + } + + // TODO: Add logic to mark these repository as System Repository once thats merged. + private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + String type = validateAttributeNonNull( + node, + String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name) + ); + Map settingsMap = validateSettingsAttributesNonNull( + node, + String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, name) + ); + + Settings.Builder settings = Settings.builder(); + settingsMap.forEach(settings::put); + + return new RepositoryMetadata(name, type, settings.build()); + } + + private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { + String segmentRepositoryName = node.getAttributes().get(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + String translogRepositoryName = node.getAttributes().get(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + if (segmentRepositoryName.equals(translogRepositoryName)) { + return new RepositoriesMetadata(Collections.singletonList(buildRepositoryMetadata(node, segmentRepositoryName))); + } else { + List repositoryMetadataList = new ArrayList<>(); + repositoryMetadataList.add(buildRepositoryMetadata(node, segmentRepositoryName)); + repositoryMetadataList.add(buildRepositoryMetadata(node, translogRepositoryName)); + return new RepositoriesMetadata(repositoryMetadataList); + } + } + + RepositoriesMetadata getRepositoriesMetadata() { + return this.repositoriesMetadata; + } + + @Override + public int hashCode() { + // We will hash the id and repositories metadata as its highly unlikely that two nodes will have same id and + // repositories metadata but are actually different. + return Objects.hash(node.getEphemeralId(), repositoriesMetadata); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RemoteStoreNode that = (RemoteStoreNode) o; + + return this.getRepositoriesMetadata().equalsIgnoreGenerations(that.getRepositoriesMetadata()); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('{').append(this.node).append('}'); + sb.append('{').append(this.repositoriesMetadata).append('}'); + return super.toString(); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java new file mode 100644 index 0000000000000..be67355389987 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -0,0 +1,204 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.cluster.remotestore; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.action.ActionListener; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryMissingException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.function.Supplier; + +/** + * Contains all the method needed for a remote store node lifecycle. + */ +public class RemoteStoreService { + + private static final Logger logger = LogManager.getLogger(RemoteStoreService.class); + private final Supplier repositoriesService; + public static final Setting REMOTE_STORE_MIGRATION_SETTING = Setting.simpleString( + "remote_store.migration", + MigrationTypes.NOT_MIGRATING.value, + MigrationTypes::validate, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + public enum MigrationTypes { + NOT_MIGRATING("not_migrating"), + MIGRATING_TO_REMOTE_STORE("migrating_to_remote_store"), + MIGRATING_TO_HOT("migrating_to_hot"); + + public static MigrationTypes validate(String migrationType) { + try { + return MigrationTypes.valueOf(migrationType.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "[" + + migrationType + + "] migration type is not supported. " + + "Supported migration types are [" + + MigrationTypes.values().toString() + + "]" + ); + } + } + + public final String value; + + MigrationTypes(String value) { + this.value = value; + } + } + + public RemoteStoreService(Supplier repositoriesService) { + this.repositoriesService = repositoriesService; + } + + public void verifyRepository(RepositoryMetadata repositoryMetadata) { + ActionListener listener = new ActionListener<>() { + + @Override + public void onResponse(VerifyRepositoryResponse verifyRepositoryResponse) { + logger.info("Successfully verified repository : " + verifyRepositoryResponse.toString()); + } + + @Override + public void onFailure(Exception e) { + throw new IllegalStateException("Failed to finish remote store repository verification" + e.getMessage()); + } + }; + + repositoriesService.get() + .verifyRepository( + repositoryMetadata.name(), + ActionListener.delegateFailure( + listener, + (delegatedListener, verifyResponse) -> delegatedListener.onResponse( + new VerifyRepositoryResponse(verifyResponse.toArray(new DiscoveryNode[0])) + ) + ) + ); + } + + private ClusterState createRepository(RepositoryMetadata newRepositoryMetadata, ClusterState currentState) { + RepositoriesService.validate(newRepositoryMetadata.name()); + + Metadata metadata = currentState.metadata(); + Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); + RepositoriesMetadata repositories = metadata.custom(RepositoriesMetadata.TYPE); + if (repositories == null) { + Repository repository = repositoriesService.get().createRepository(newRepositoryMetadata); + logger.info( + "Remote store repository with name {} and type {} created.", + repository.getMetadata().name(), + repository.getMetadata().type() + ); + repositories = new RepositoriesMetadata(Collections.singletonList(newRepositoryMetadata)); + } else { + List repositoriesMetadata = new ArrayList<>(repositories.repositories().size() + 1); + + for (RepositoryMetadata repositoryMetadata : repositories.repositories()) { + if (repositoryMetadata.name().equals(newRepositoryMetadata.name())) { + if (newRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)) { + return new ClusterState.Builder(currentState).build(); + } else { + throw new IllegalStateException( + "new repository metadata [" + + newRepositoryMetadata + + "] supplied by joining node is different from existing repository metadata [" + + repositoryMetadata + + "]." + ); + } + } else { + repositoriesMetadata.add(repositoryMetadata); + } + } + Repository repository = repositoriesService.get().createRepository(newRepositoryMetadata); + logger.info( + "Remote store repository with name {} and type {} created", + repository.getMetadata().name(), + repository.getMetadata().type() + ); + repositoriesMetadata.add(newRepositoryMetadata); + repositories = new RepositoriesMetadata(repositoriesMetadata); + } + mdBuilder.putCustom(RepositoriesMetadata.TYPE, repositories); + return ClusterState.builder(currentState).metadata(mdBuilder).build(); + } + + private boolean isRepositoryCreated(RepositoryMetadata repositoryMetadata) { + try { + repositoriesService.get().repository(repositoryMetadata.name()); + return true; + } catch (RepositoryMissingException e) { + return false; + } + } + + private boolean isRepositoryAddedInClusterState(RepositoryMetadata repositoryMetadata, ClusterState currentState) { + RepositoriesMetadata repositoriesMetadata = currentState.metadata().custom(RepositoriesMetadata.TYPE); + if (repositoriesMetadata == null) { + return false; + } + for (RepositoryMetadata existingRepositoryMetadata : repositoriesMetadata.repositories()) { + existingRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata); + return true; + } + return false; + } + + private ClusterState createOrVerifyRepository(RepositoriesMetadata repositoriesMetadata, ClusterState currentState) { + ClusterState newState = ClusterState.builder(currentState).build(); + for (RepositoryMetadata repositoryMetadata : repositoriesMetadata.repositories()) { + if (isRepositoryCreated(repositoryMetadata)) { + verifyRepository(repositoryMetadata); + } else { + if (!isRepositoryAddedInClusterState(repositoryMetadata, currentState)) { + newState = ClusterState.builder(createRepository(repositoryMetadata, newState)).build(); + // verifyRepository(repositoryMetadata); + } + } + } + return newState; + } + + public ClusterState joinCluster(RemoteStoreNode joiningRemoteStoreNode, ClusterState currentState) { + List existingNodes = new ArrayList<>(currentState.nodes().getNodes().values()); + if (existingNodes.isEmpty()) { + return currentState; + } + ClusterState.Builder newState = ClusterState.builder(currentState); + if (existingNodes.get(0).isRemoteStoreNode()) { + RemoteStoreNode existingRemoteStoreNode = new RemoteStoreNode(existingNodes.get(0)); + if (existingRemoteStoreNode.equals(joiningRemoteStoreNode)) { + newState = ClusterState.builder(createOrVerifyRepository(joiningRemoteStoreNode.getRepositoriesMetadata(), currentState)); + } + } else { + throw new IllegalStateException( + "a remote store node [" + joiningRemoteStoreNode + "] is trying to join a non remote store cluster." + ); + } + return newState.build(); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 1559e77e1cf2d..1a2ec948d0de9 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -182,6 +183,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private Optional currentPublication = Optional.empty(); private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; + private final RemoteStoreService remoteStoreService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -203,7 +205,8 @@ public Coordinator( RerouteService rerouteService, ElectionStrategy electionStrategy, NodeHealthService nodeHealthService, - PersistedStateRegistry persistedStateRegistry + PersistedStateRegistry persistedStateRegistry, + RemoteStoreService remoteStoreService ) { this.settings = settings; this.transportService = transportService; @@ -225,7 +228,8 @@ public Coordinator( rerouteService, nodeHealthService, this::onNodeCommissionStatusChange, - namedWriteableRegistry + namedWriteableRegistry, + remoteStoreService ); this.persistedStateSupplier = persistedStateSupplier; this.noClusterManagerBlockService = new NoClusterManagerBlockService(settings, clusterSettings); @@ -290,6 +294,7 @@ public Coordinator( this.nodeHealthService = nodeHealthService; this.persistedStateRegistry = persistedStateRegistry; this.localNodeCommissioned = true; + this.remoteStoreService = remoteStoreService; } private ClusterFormationState getClusterFormationState() { @@ -608,6 +613,8 @@ private void handleJoinRequest(JoinRequest joinRequest, JoinHelper.JoinCallback // we are checking source node commission status here to reject any join request coming from a decommissioned node // even before executing the join task to fail fast JoinTaskExecutor.ensureNodeCommissioned(joinRequest.getSourceNode(), stateForJoinValidation.metadata()); + + JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joinRequest.getSourceNode(), stateForJoinValidation); } sendValidateJoinRequest(stateForJoinValidation, joinRequest, joinCallback); } else { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 9a1f14295fad8..5f8b332525cb1 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateTaskListener; @@ -143,7 +144,8 @@ public class JoinHelper { RerouteService rerouteService, NodeHealthService nodeHealthService, Consumer nodeCommissioned, - NamedWriteableRegistry namedWriteableRegistry + NamedWriteableRegistry namedWriteableRegistry, + RemoteStoreService remoteStoreService ) { this.clusterManagerService = clusterManagerService; this.transportService = transportService; @@ -152,7 +154,13 @@ public class JoinHelper { this.nodeCommissioned = nodeCommissioned; this.namedWriteableRegistry = namedWriteableRegistry; - this.joinTaskExecutorGenerator = () -> new JoinTaskExecutor(settings, allocationService, logger, rerouteService) { + this.joinTaskExecutorGenerator = () -> new JoinTaskExecutor( + settings, + allocationService, + logger, + rerouteService, + remoteStoreService + ) { private final long term = currentTermSupplier.getAsLong(); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 564819a70111d..c879898e9effb 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -33,6 +33,8 @@ import org.apache.logging.log4j.Logger; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; import org.opensearch.cluster.NotClusterManagerException; @@ -74,6 +76,8 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor execute(ClusterState currentState, List jo // processing any joins Map joiniedNodeNameIds = new HashMap<>(); for (final Task joinTask : joiningNodes) { + final DiscoveryNode node = joinTask.node(); if (joinTask.isBecomeClusterManagerTask() || joinTask.isFinishElectionTask()) { // noop - } else if (currentNodes.nodeExistsWithSameRoles(joinTask.node())) { - logger.debug("received a join request for an existing node [{}]", joinTask.node()); + } else if (currentNodes.nodeExistsWithSameRoles(node)) { + logger.debug("received a join request for an existing node [{}]", node); + if (node.isRemoteStoreNode()) { + /** joinCluster for remote store node is invoked here as elect leader task can have same node + * present in join task as well as current node. We want the repositories to be registered during + * first node join. See + * {@link org.opensearch.gateway.GatewayMetaState#prepareInitialClusterState(TransportService, ClusterService, ClusterState)} **/ + newState = ClusterState.builder(remoteStoreService.joinCluster(new RemoteStoreNode(node), currentState)); + } } else { - final DiscoveryNode node = joinTask.node(); + if (node.isRemoteStoreNode()) { + newState = ClusterState.builder(remoteStoreService.joinCluster(new RemoteStoreNode(node), currentState)); + } try { if (enforceMajorVersion) { ensureMajorVersionBarrier(node.getVersion(), minClusterNodeVersion); @@ -187,6 +208,8 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // we have added the same check in handleJoinRequest method and adding it here as this method // would guarantee that a decommissioned node would never be able to join the cluster and ensures correctness ensureNodeCommissioned(node, currentState.metadata()); + + ensureRemoteStoreNodesCompatibility(node, currentState); nodesBuilder.add(node); nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); @@ -422,6 +445,61 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) } } + /** + * The method ensures two conditions - + * 1. The joining node is remote store if it is joining a remote store cluster. + * 2. The joining node is non-remote store if it is joining a non-remote store cluster. + * A remote store node is the one which holds all the remote store attributes and a remote store cluster is + * the one which has only homogeneous remote store nodes with same node attributes + * + * TODO: When we support migration from remote store cluster to non remote store and vice versa the migration + * setting {@link RemoteStoreService::REMOTE_STORE_MIGRATION_SETTING} will be help determine if a non + * remote store node is allowed to join the remote store cluster and vice versa. + */ + public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, ClusterState currentState) { + List existingNodes = new ArrayList<>(currentState.getNodes().getNodes().values()); + + // If there are no node in the cluster state we will No op the compatibility check as at this point we cannot + // determine if this is a remote store cluster or non-remote store cluster. + if (existingNodes.isEmpty()) { + return; + } + + // TODO: The below check is valid till we support migration, once we start supporting migration a remote + // store node will be able to join a non remote store cluster and vice versa. #7986 + if (RemoteStoreService.MigrationTypes.NOT_MIGRATING.value.equals( + RemoteStoreService.REMOTE_STORE_MIGRATION_SETTING.get(currentState.metadata().settings()) + )) { + DiscoveryNode existingNode = existingNodes.get(0); + if (joiningNode.isRemoteStoreNode()) { + if (existingNode.isRemoteStoreNode()) { + RemoteStoreNode joiningRemoteStoreNode = new RemoteStoreNode(joiningNode); + RemoteStoreNode existingRemoteStoreNode = new RemoteStoreNode(existingNode); + if (existingRemoteStoreNode.equals(joiningRemoteStoreNode) == false) { + throw new IllegalStateException( + "a remote store node [" + + joiningNode + + "] is trying to join a remote store cluster with incompatible node attributes in " + + "comparison with existing node [" + + existingNode + + "]." + ); + } + } else { + throw new IllegalStateException( + "a remote store node [" + joiningNode + "] is trying to join a non remote store cluster." + ); + } + } else { + if (existingNode.isRemoteStoreNode()) { + throw new IllegalStateException( + "a non remote store node [" + joiningNode + "] is trying to join a remote store cluster." + ); + } + } + } + } + public static Collection> addBuiltInJoinValidators( Collection> onJoinValidators ) { @@ -430,6 +508,7 @@ public static Collection> addBuiltInJoin ensureNodesCompatibility(node.getVersion(), state.getNodes()); ensureIndexCompatibility(node.getVersion(), state.getMetadata()); ensureNodeCommissioned(node, state.getMetadata()); + ensureRemoteStoreNodesCompatibility(node, state); }); validators.addAll(onJoinValidators); return Collections.unmodifiableCollection(validators); diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index a04b0d9de912d..de4f2ab74a7bf 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -60,6 +60,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; /** @@ -460,6 +461,15 @@ public boolean isSearchNode() { return roles.contains(DiscoveryNodeRole.SEARCH_ROLE); } + /** + * Returns whether the node is a remote store node. + * + * @return true if the node contains remote store node attributes, false otherwise + */ + public boolean isRemoteStoreNode() { + return this.getAttributes().keySet().stream().anyMatch(key -> key.startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)); + } + /** * Returns a set of all the roles that the node has. The roles are returned in sorted order by the role name. *

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 05938914b019f..2336bf72e29a7 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -33,6 +33,7 @@ import org.apache.logging.log4j.LogManager; import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.action.admin.indices.close.TransportCloseIndexAction; import org.opensearch.action.search.CreatePitController; import org.opensearch.action.search.TransportSearchAction; @@ -670,6 +671,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING + RemoteStoreService.REMOTE_STORE_MIGRATION_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index 58d8fe2e17fcf..dd97ba1a4db3a 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.coordination.ElectionStrategy; @@ -131,7 +132,8 @@ public DiscoveryModule( GatewayMetaState gatewayMetaState, RerouteService rerouteService, NodeHealthService nodeHealthService, - PersistedStateRegistry persistedStateRegistry + PersistedStateRegistry persistedStateRegistry, + RemoteStoreService remoteStoreService ) { final Collection> joinValidators = new ArrayList<>(); final Map> hostProviders = new HashMap<>(); @@ -208,7 +210,8 @@ public DiscoveryModule( rerouteService, electionStrategy, nodeHealthService, - persistedStateRegistry + persistedStateRegistry, + remoteStoreService ); } else { throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]"); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index fdf8b616ccb6c..488e95909aca4 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -43,6 +43,7 @@ import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; import org.opensearch.action.search.SearchPhaseController; @@ -985,6 +986,8 @@ protected Node( ); clusterInfoService.addListener(diskThresholdMonitor::onNewInfo); + final RemoteStoreService remoteStoreService = new RemoteStoreService(repositoriesServiceReference::get); + final DiscoveryModule discoveryModule = new DiscoveryModule( settings, threadPool, @@ -1000,7 +1003,8 @@ protected Node( gatewayMetaState, rerouteService, fsHealthService, - persistedStateRegistry + persistedStateRegistry, + remoteStoreService ); final SearchPipelineService searchPipelineService = new SearchPipelineService( clusterService, diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 5c2a19965c532..256c4694dfd3e 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -601,6 +601,13 @@ private void archiveRepositoryStats(Repository repository, long clusterStateVers } } + /** + * Creates repository holder. This method starts the non-internal repository + */ + public Repository createRepository(RepositoryMetadata repositoryMetadata) { + return this.createRepository(repositoryMetadata, typesRegistry); + } + /** * Creates repository holder. This method starts the repository */ @@ -625,7 +632,7 @@ private Repository createRepository(RepositoryMetadata repositoryMetadata, Map(repositoriesService)::get); + nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger); - joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, (s, p, r) -> {}); + joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, (s, p, r) -> {}, remoteStoreService); } public ClusterState createIndex(ClusterState state, CreateIndexRequest request) { From aaa8fd1e11fb26b9732de88da57a2cb7817adc35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sat, 26 Aug 2023 22:10:39 +0530 Subject: [PATCH 02/34] Code changes to do the repository verification at the time of node bootstrap. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../cluster/remotestore/RemoteStoreNode.java | 34 ++--- .../remotestore/RemoteStoreService.java | 133 ++++++------------ .../coordination/JoinTaskExecutor.java | 21 +-- .../cluster/node/DiscoveryNode.java | 27 ++++ .../common/settings/ClusterSettings.java | 4 +- .../main/java/org/opensearch/node/Node.java | 24 +++- 6 files changed, 118 insertions(+), 125 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java index 2006d46edc443..263b13645e605 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java @@ -22,55 +22,49 @@ import java.util.stream.Collectors; /** - * This is an extension of {@Code DiscoveryNode} which provides an abstraction for validating and storing information - * specific to remote backed storage nodes. + * This is an abstraction for validating and storing information specific to remote backed storage nodes. * * @opensearch.internal */ -public class RemoteStoreNode extends DiscoveryNode { +public class RemoteStoreNode { public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; - private final DiscoveryNode node; private final RepositoriesMetadata repositoriesMetadata; + private final DiscoveryNode node; /** * Creates a new {@link RemoteStoreNode} */ public RemoteStoreNode(DiscoveryNode node) { - super(node.getName(), node.getId(), node.getAddress(), node.getAttributes(), node.getRoles(), node.getVersion()); this.node = node; - this.repositoriesMetadata = buildRepositoriesMetadata(node); + this.repositoriesMetadata = buildRepositoriesMetadata(); } - private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) { + private String validateAttributeNonNull(String attributeKey) { String attributeValue = node.getAttributes().get(attributeKey); if (attributeValue == null || attributeValue.isEmpty()) { - throw new IllegalStateException("joining node [" + node + "] doesn't have the node attribute [" + attributeKey + "]."); + throw new IllegalStateException("joining node [" + this + "] doesn't have the node attribute [" + attributeKey + "]."); } return attributeValue; } - private Map validateSettingsAttributesNonNull(DiscoveryNode node, String settingsAttributeKeyPrefix) { + private Map validateSettingsAttributesNonNull(String settingsAttributeKeyPrefix) { return node.getAttributes() .keySet() .stream() .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) - .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(node, key))); + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(key))); } // TODO: Add logic to mark these repository as System Repository once thats merged. - private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { - String type = validateAttributeNonNull( - node, - String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name) - ); + private RepositoryMetadata buildRepositoryMetadata(String name) { + String type = validateAttributeNonNull(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); Map settingsMap = validateSettingsAttributesNonNull( - node, String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, name) ); @@ -80,15 +74,15 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na return new RepositoryMetadata(name, type, settings.build()); } - private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { + private RepositoriesMetadata buildRepositoriesMetadata() { String segmentRepositoryName = node.getAttributes().get(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); String translogRepositoryName = node.getAttributes().get(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); if (segmentRepositoryName.equals(translogRepositoryName)) { - return new RepositoriesMetadata(Collections.singletonList(buildRepositoryMetadata(node, segmentRepositoryName))); + return new RepositoriesMetadata(Collections.singletonList(buildRepositoryMetadata(segmentRepositoryName))); } else { List repositoryMetadataList = new ArrayList<>(); - repositoryMetadataList.add(buildRepositoryMetadata(node, segmentRepositoryName)); - repositoryMetadataList.add(buildRepositoryMetadata(node, translogRepositoryName)); + repositoryMetadataList.add(buildRepositoryMetadata(segmentRepositoryName)); + repositoryMetadataList.add(buildRepositoryMetadata(translogRepositoryName)); return new RepositoriesMetadata(repositoryMetadataList); } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index be67355389987..fc7f57461bb88 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -20,7 +20,6 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; -import org.opensearch.repositories.RepositoryMissingException; import java.util.ArrayList; import java.util.Collections; @@ -35,29 +34,29 @@ public class RemoteStoreService { private static final Logger logger = LogManager.getLogger(RemoteStoreService.class); private final Supplier repositoriesService; - public static final Setting REMOTE_STORE_MIGRATION_SETTING = Setting.simpleString( - "remote_store.migration", - MigrationTypes.NOT_MIGRATING.value, - MigrationTypes::validate, + public static final Setting REMOTE_STORE_COMPATIBILITY_MODE_SETTING = Setting.simpleString( + "remote_store.compatibility_mode", + CompatibilityMode.ALLOW_ONLY_REMOTE_STORE_NODES.value, + CompatibilityMode::validate, Setting.Property.Dynamic, Setting.Property.NodeScope ); - public enum MigrationTypes { - NOT_MIGRATING("not_migrating"), - MIGRATING_TO_REMOTE_STORE("migrating_to_remote_store"), + public enum CompatibilityMode { + ALLOW_ONLY_REMOTE_STORE_NODES("allow_only_remote_store_nodes"), + ALLOW_ALL_NODES("allow_all_nodes"), MIGRATING_TO_HOT("migrating_to_hot"); - public static MigrationTypes validate(String migrationType) { + public static CompatibilityMode validate(String compatibilityMode) { try { - return MigrationTypes.valueOf(migrationType.toUpperCase(Locale.ROOT)); + return CompatibilityMode.valueOf(compatibilityMode.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "[" - + migrationType - + "] migration type is not supported. " - + "Supported migration types are [" - + MigrationTypes.values().toString() + + compatibilityMode + + "] compatibility mode is not supported. " + + "supported modes are [" + + CompatibilityMode.values().toString() + "]" ); } @@ -65,7 +64,7 @@ public static MigrationTypes validate(String migrationType) { public final String value; - MigrationTypes(String value) { + CompatibilityMode(String value) { this.value = value; } } @@ -74,7 +73,7 @@ public RemoteStoreService(Supplier repositoriesService) { this.repositoriesService = repositoriesService; } - public void verifyRepository(RepositoryMetadata repositoryMetadata) { + public void verifyRepository(List repositories) { ActionListener listener = new ActionListener<>() { @Override @@ -88,31 +87,40 @@ public void onFailure(Exception e) { } }; - repositoriesService.get() - .verifyRepository( - repositoryMetadata.name(), - ActionListener.delegateFailure( - listener, - (delegatedListener, verifyResponse) -> delegatedListener.onResponse( - new VerifyRepositoryResponse(verifyResponse.toArray(new DiscoveryNode[0])) + for (Repository repository : repositories) { + repositoriesService.get() + .verifyRepository( + repository.getMetadata().name(), + ActionListener.delegateFailure( + listener, + (delegatedListener, verifyResponse) -> delegatedListener.onResponse( + new VerifyRepositoryResponse(verifyResponse.toArray(new DiscoveryNode[0])) + ) ) - ) - ); + ); + } } - private ClusterState createRepository(RepositoryMetadata newRepositoryMetadata, ClusterState currentState) { - RepositoriesService.validate(newRepositoryMetadata.name()); - - Metadata metadata = currentState.metadata(); - Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); - RepositoriesMetadata repositories = metadata.custom(RepositoriesMetadata.TYPE); - if (repositories == null) { - Repository repository = repositoriesService.get().createRepository(newRepositoryMetadata); + public List createRepositories(RemoteStoreNode node) { + List repositories = new ArrayList<>(); + for (RepositoryMetadata repositoryMetadata : node.getRepositoriesMetadata().repositories()) { + RepositoriesService.validate(repositoryMetadata.name()); + Repository repository = repositoriesService.get().createRepository(repositoryMetadata); logger.info( "Remote store repository with name {} and type {} created.", repository.getMetadata().name(), repository.getMetadata().type() ); + repositories.add(repository); + } + return repositories; + } + + private ClusterState updateRepositoryMetadata(RepositoryMetadata newRepositoryMetadata, ClusterState currentState) { + Metadata metadata = currentState.metadata(); + Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); + RepositoriesMetadata repositories = metadata.custom(RepositoriesMetadata.TYPE); + if (repositories == null) { repositories = new RepositoriesMetadata(Collections.singletonList(newRepositoryMetadata)); } else { List repositoriesMetadata = new ArrayList<>(repositories.repositories().size() + 1); @@ -134,12 +142,6 @@ private ClusterState createRepository(RepositoryMetadata newRepositoryMetadata, repositoriesMetadata.add(repositoryMetadata); } } - Repository repository = repositoriesService.get().createRepository(newRepositoryMetadata); - logger.info( - "Remote store repository with name {} and type {} created", - repository.getMetadata().name(), - repository.getMetadata().type() - ); repositoriesMetadata.add(newRepositoryMetadata); repositories = new RepositoriesMetadata(repositoriesMetadata); } @@ -147,58 +149,11 @@ private ClusterState createRepository(RepositoryMetadata newRepositoryMetadata, return ClusterState.builder(currentState).metadata(mdBuilder).build(); } - private boolean isRepositoryCreated(RepositoryMetadata repositoryMetadata) { - try { - repositoriesService.get().repository(repositoryMetadata.name()); - return true; - } catch (RepositoryMissingException e) { - return false; - } - } - - private boolean isRepositoryAddedInClusterState(RepositoryMetadata repositoryMetadata, ClusterState currentState) { - RepositoriesMetadata repositoriesMetadata = currentState.metadata().custom(RepositoriesMetadata.TYPE); - if (repositoriesMetadata == null) { - return false; - } - for (RepositoryMetadata existingRepositoryMetadata : repositoriesMetadata.repositories()) { - existingRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata); - return true; - } - return false; - } - - private ClusterState createOrVerifyRepository(RepositoriesMetadata repositoriesMetadata, ClusterState currentState) { + public ClusterState updateClusterStateRepositoriesMetadata(RemoteStoreNode node, ClusterState currentState) { ClusterState newState = ClusterState.builder(currentState).build(); - for (RepositoryMetadata repositoryMetadata : repositoriesMetadata.repositories()) { - if (isRepositoryCreated(repositoryMetadata)) { - verifyRepository(repositoryMetadata); - } else { - if (!isRepositoryAddedInClusterState(repositoryMetadata, currentState)) { - newState = ClusterState.builder(createRepository(repositoryMetadata, newState)).build(); - // verifyRepository(repositoryMetadata); - } - } + for (RepositoryMetadata newRepositoryMetadata : node.getRepositoriesMetadata().repositories()) { + newState = updateRepositoryMetadata(newRepositoryMetadata, newState); } return newState; } - - public ClusterState joinCluster(RemoteStoreNode joiningRemoteStoreNode, ClusterState currentState) { - List existingNodes = new ArrayList<>(currentState.nodes().getNodes().values()); - if (existingNodes.isEmpty()) { - return currentState; - } - ClusterState.Builder newState = ClusterState.builder(currentState); - if (existingNodes.get(0).isRemoteStoreNode()) { - RemoteStoreNode existingRemoteStoreNode = new RemoteStoreNode(existingNodes.get(0)); - if (existingRemoteStoreNode.equals(joiningRemoteStoreNode)) { - newState = ClusterState.builder(createOrVerifyRepository(joiningRemoteStoreNode.getRepositoriesMetadata(), currentState)); - } - } else { - throw new IllegalStateException( - "a remote store node [" + joiningRemoteStoreNode + "] is trying to join a non remote store cluster." - ); - } - return newState.build(); - } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index c879898e9effb..da1c2c03cd342 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -61,6 +61,8 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode.ALLOW_ONLY_REMOTE_STORE_NODES; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -191,12 +193,11 @@ public ClusterTasksResult execute(ClusterState currentState, List jo * present in join task as well as current node. We want the repositories to be registered during * first node join. See * {@link org.opensearch.gateway.GatewayMetaState#prepareInitialClusterState(TransportService, ClusterService, ClusterState)} **/ - newState = ClusterState.builder(remoteStoreService.joinCluster(new RemoteStoreNode(node), currentState)); + newState = ClusterState.builder( + remoteStoreService.updateClusterStateRepositoriesMetadata(new RemoteStoreNode(node), currentState) + ); } } else { - if (node.isRemoteStoreNode()) { - newState = ClusterState.builder(remoteStoreService.joinCluster(new RemoteStoreNode(node), currentState)); - } try { if (enforceMajorVersion) { ensureMajorVersionBarrier(node.getVersion(), minClusterNodeVersion); @@ -217,6 +218,11 @@ public ClusterTasksResult execute(ClusterState currentState, List jo if (node.isClusterManagerNode()) { joiniedNodeNameIds.put(node.getName(), node.getId()); } + if (node.isRemoteStoreNode()) { + newState = ClusterState.builder( + remoteStoreService.updateClusterStateRepositoriesMetadata(new RemoteStoreNode(node), currentState) + ); + } } catch (IllegalArgumentException | IllegalStateException | NodeDecommissionedException e) { results.failure(joinTask, e); continue; @@ -466,10 +472,9 @@ public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode } // TODO: The below check is valid till we support migration, once we start supporting migration a remote - // store node will be able to join a non remote store cluster and vice versa. #7986 - if (RemoteStoreService.MigrationTypes.NOT_MIGRATING.value.equals( - RemoteStoreService.REMOTE_STORE_MIGRATION_SETTING.get(currentState.metadata().settings()) - )) { + // store node will be able to join a non remote store cluster and vice versa. #7986 + String remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(currentState.metadata().settings()); + if (ALLOW_ONLY_REMOTE_STORE_NODES.value.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index de4f2ab74a7bf..57ceeb1f61720 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -33,6 +33,8 @@ package org.opensearch.cluster.node; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; @@ -44,11 +46,13 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; +import org.opensearch.repositories.Repository; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -282,6 +286,29 @@ public static DiscoveryNode createLocal(Settings settings, TransportAddress publ return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress, attributes, roles, Version.CURRENT); } + /** Creates a DiscoveryNode representing the local node and verifies the repository. */ + public static DiscoveryNode createLocal( + Settings settings, + TransportAddress publishAddress, + String nodeId, + RemoteStoreService remoteStoreService + ) { + Map attributes = Node.NODE_ATTRIBUTES.getAsMap(settings); + Set roles = getRolesFromSettings(settings); + DiscoveryNode discoveryNode = new DiscoveryNode( + Node.NODE_NAME_SETTING.get(settings), + nodeId, + publishAddress, + attributes, + roles, + Version.CURRENT + ); + RemoteStoreNode remoteStoreNode = new RemoteStoreNode(discoveryNode); + List repositories = remoteStoreService.createRepositories(remoteStoreNode); + remoteStoreService.verifyRepository(repositories); + return discoveryNode; + } + /** extract node roles from the given settings */ public static Set getRolesFromSettings(final Settings settings) { if (NODE_ROLES_SETTING.exists(settings)) { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 2336bf72e29a7..1077a103fdc4b 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -670,8 +670,8 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, - RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING - RemoteStoreService.REMOTE_STORE_MIGRATION_SETTING + RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING, + RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 488e95909aca4..35c55608f886a 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -43,6 +43,7 @@ import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; @@ -523,7 +524,10 @@ protected Node( .collect(Collectors.toCollection(LinkedHashSet::new)) ); resourcesToClose.add(nodeEnvironment); - localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId()); + + final SetOnce repositoriesServiceReference = new SetOnce<>(); + final RemoteStoreService remoteStoreService = new RemoteStoreService(repositoriesServiceReference::get); + localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId(), remoteStoreService); final List> executorBuilders = pluginsService.getExecutorBuilders(settings); @@ -592,7 +596,6 @@ protected Node( client ); - final SetOnce repositoriesServiceReference = new SetOnce<>(); final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client); final UsageService usageService = new UsageService(); @@ -986,8 +989,6 @@ protected Node( ); clusterInfoService.addListener(diskThresholdMonitor::onNewInfo); - final RemoteStoreService remoteStoreService = new RemoteStoreService(repositoriesServiceReference::get); - final DiscoveryModule discoveryModule = new DiscoveryModule( settings, threadPool, @@ -1743,15 +1744,26 @@ private static class LocalNodeFactory implements Function localNode = new SetOnce<>(); private final String persistentNodeId; private final Settings settings; + private final RemoteStoreService remoteStoreService; - private LocalNodeFactory(Settings settings, String persistentNodeId) { + private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStoreService remoteStoreService) { this.persistentNodeId = persistentNodeId; this.settings = settings; + this.remoteStoreService = remoteStoreService; } @Override public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { - localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId)); + if (Node.NODE_ATTRIBUTES.getAsMap(settings) + .keySet() + .stream() + .anyMatch(key -> key.startsWith(RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX))) { + localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId)); + } else { + localNode.set( + DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId, remoteStoreService) + ); + } return localNode.get(); } From ee7d06a29b803d2d0449bdac126dee05013924cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 27 Aug 2023 02:29:08 +0530 Subject: [PATCH 03/34] Fixing repository verification logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteStoreService.java | 60 ++++++++++--------- .../cluster/node/DiscoveryNode.java | 2 +- .../main/java/org/opensearch/node/Node.java | 12 ++-- .../indices/cluster/ClusterStateChanges.java | 2 +- 4 files changed, 40 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index fc7f57461bb88..4254b3efd79ed 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -10,21 +10,25 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Setting; -import org.opensearch.core.action.ActionListener; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; +import org.opensearch.repositories.RepositoryVerificationException; +import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; /** @@ -34,6 +38,7 @@ public class RemoteStoreService { private static final Logger logger = LogManager.getLogger(RemoteStoreService.class); private final Supplier repositoriesService; + private final ThreadPool threadPool; public static final Setting REMOTE_STORE_COMPATIBILITY_MODE_SETTING = Setting.simpleString( "remote_store.compatibility_mode", CompatibilityMode.ALLOW_ONLY_REMOTE_STORE_NODES.value, @@ -44,8 +49,7 @@ public class RemoteStoreService { public enum CompatibilityMode { ALLOW_ONLY_REMOTE_STORE_NODES("allow_only_remote_store_nodes"), - ALLOW_ALL_NODES("allow_all_nodes"), - MIGRATING_TO_HOT("migrating_to_hot"); + ALLOW_ALL_NODES("allow_all_nodes"); public static CompatibilityMode validate(String compatibilityMode) { try { @@ -69,35 +73,35 @@ public static CompatibilityMode validate(String compatibilityMode) { } } - public RemoteStoreService(Supplier repositoriesService) { + public RemoteStoreService(Supplier repositoriesService, ThreadPool threadPool) { this.repositoriesService = repositoriesService; + this.threadPool = threadPool; } - public void verifyRepository(List repositories) { - ActionListener listener = new ActionListener<>() { - - @Override - public void onResponse(VerifyRepositoryResponse verifyRepositoryResponse) { - logger.info("Successfully verified repository : " + verifyRepositoryResponse.toString()); - } + public void verifyRepository(List repositories, DiscoveryNode localNode) { + for (Repository repository : repositories) { + String verificationToken = repository.startVerification(); + ExecutorService executor = threadPool.executor(ThreadPool.Names.GENERIC); + executor.execute(() -> { + try { + repository.verify(verificationToken, localNode); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); + throw new RepositoryVerificationException(repository.getMetadata().name(), e.getMessage()); + } + }); - @Override - public void onFailure(Exception e) { - throw new IllegalStateException("Failed to finish remote store repository verification" + e.getMessage()); + // TODO: See if using listener here which is async makes sense, made this sync as + // we need the repository registration for remote store node to be completed before the bootstrap + // completes. + try { + if(executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) { + throw new RepositoryVerificationException(repository.getMetadata().name(), "could not complete " + + "repository verification within timeout."); + } + } catch (InterruptedException e) { + throw new RepositoryVerificationException(repository.getMetadata().name(), e.getMessage()); } - }; - - for (Repository repository : repositories) { - repositoriesService.get() - .verifyRepository( - repository.getMetadata().name(), - ActionListener.delegateFailure( - listener, - (delegatedListener, verifyResponse) -> delegatedListener.onResponse( - new VerifyRepositoryResponse(verifyResponse.toArray(new DiscoveryNode[0])) - ) - ) - ); } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 57ceeb1f61720..0fb8a54209a1b 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -305,7 +305,7 @@ public static DiscoveryNode createLocal( ); RemoteStoreNode remoteStoreNode = new RemoteStoreNode(discoveryNode); List repositories = remoteStoreService.createRepositories(remoteStoreNode); - remoteStoreService.verifyRepository(repositories); + remoteStoreService.verifyRepository(repositories, discoveryNode); return discoveryNode; } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 35c55608f886a..a088cb743bdb5 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -525,14 +525,14 @@ protected Node( ); resourcesToClose.add(nodeEnvironment); - final SetOnce repositoriesServiceReference = new SetOnce<>(); - final RemoteStoreService remoteStoreService = new RemoteStoreService(repositoriesServiceReference::get); - localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId(), remoteStoreService); - final List> executorBuilders = pluginsService.getExecutorBuilders(settings); runnableTaskListener = new AtomicReference<>(); final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, executorBuilders.toArray(new ExecutorBuilder[0])); + + final SetOnce repositoriesServiceReference = new SetOnce<>(); + final RemoteStoreService remoteStoreService = new RemoteStoreService(repositoriesServiceReference::get, threadPool); + localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId(), remoteStoreService); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); resourcesToClose.add(resourceWatcherService); @@ -1758,11 +1758,11 @@ public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { .keySet() .stream() .anyMatch(key -> key.startsWith(RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX))) { - localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId)); - } else { localNode.set( DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId, remoteStoreService) ); + } else { + localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId)); } return localNode.get(); } diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index a928c3b055282..06b604152bf11 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -376,7 +376,7 @@ public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version m threadPool ); - remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get); + remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger); joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, (s, p, r) -> {}, remoteStoreService); From 58466a7e43962432d0c06f7db0f7bee851e259bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 27 Aug 2023 02:53:24 +0530 Subject: [PATCH 04/34] Updating javadoc and RepoRegistration IT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../RemoteStoreBaseIntegTestCase.java | 41 +++++++ .../RemoteStoreRepositoryRegistrationIT.java | 103 ++++++++++++++++++ .../cluster/remotestore/RemoteStoreNode.java | 1 - .../remotestore/RemoteStoreService.java | 17 ++- 4 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 15d4b8d1de58a..e021226fb3d18 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -34,9 +34,14 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; @@ -55,6 +60,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected Path absolutePath; protected Path absolutePath2; + protected Settings nodeAttributesSettings; private final List documentKeys = List.of( randomAlphaOfLength(5), randomAlphaOfLength(5), @@ -108,9 +114,13 @@ protected boolean addMockInternalEngine() { @Override protected Settings nodeSettings(int nodeOrdinal) { + if (nodeAttributesSettings == null) { + nodeAttributesSettings = remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME); + } return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) + .put(nodeAttributesSettings) .build(); } @@ -181,6 +191,36 @@ public static Settings remoteStoreClusterSettings(String segmentRepoName, String return settingsBuilder.build(); } + public Settings remoteStoreNodeAttributes(String segmentRepoName, String translogRepoName) { + absolutePath = randomRepoPath().toAbsolutePath(); + absolutePath2 = randomRepoPath().toAbsolutePath(); + if (segmentRepoName.equals(translogRepoName)) { + absolutePath2 = absolutePath; + } + return Settings.builder() + .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) + .put( + String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, segmentRepoName), + "fs" + ) + .put( + String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, segmentRepoName) + + "location", + absolutePath.toString() + ) + .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) + .put( + String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, translogRepoName), + "fs" + ) + .put( + String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName) + + "location", + absolutePath2.toString() + ) + .build(); + } + private Settings defaultIndexSettings() { return Settings.builder() .put(super.indexSettings()) @@ -236,6 +276,7 @@ protected void setupRepo(boolean startDedicatedClusterManager) { @After public void teardown() { + nodeAttributesSettings = null; assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java new file mode 100644 index 0000000000000..87f1b83ae0dd1 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -0,0 +1,103 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.remotestore; + +import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.transport.MockTransportService; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0) +public class RemoteStoreRepositoryRegistrationIT extends RemoteStoreBaseIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(MockTransportService.TestPlugin.class); + } + + private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + Map nodeAttributes = node.getAttributes(); + String type = nodeAttributes.get(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); + + String settingsAttributeKeyPrefix = String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, name); + Map settingsMap = node.getAttributes() + .keySet() + .stream() + .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> node.getAttributes().get(key))); + + Settings.Builder settings = Settings.builder(); + settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + + return new RepositoryMetadata(name, type, settings.build()); + } + + private void assertRemoteStoreRepositoryOnAllNodes() { + RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) + .state() + .metadata() + .custom(RepositoriesMetadata.TYPE); + RepositoryMetadata actualSegmentRepository = repositories.repository(REPOSITORY_NAME); + RepositoryMetadata actualTranslogRepository = repositories.repository(REPOSITORY_2_NAME); + + for (String nodeName : internalCluster().getNodeNames()) { + ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); + DiscoveryNode node = clusterService.localNode(); + RepositoryMetadata expectedSegmentRepository = buildRepositoryMetadata(node, REPOSITORY_NAME); + RepositoryMetadata expectedTranslogRepository = buildRepositoryMetadata(node, REPOSITORY_2_NAME); + assertTrue(actualSegmentRepository.equalsIgnoreGenerations(expectedSegmentRepository)); + assertTrue(actualTranslogRepository.equalsIgnoreGenerations(expectedTranslogRepository)); + } + } + + public void testSingleNodeClusterRepositoryRegistration() { + internalCluster().startClusterManagerOnlyNode(remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME)); + ensureStableCluster(1); + + assertRemoteStoreRepositoryOnAllNodes(); + } + + public void testMultiNodeClusterRepositoryRegistration() { + Settings clusterSettings = remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME); + internalCluster().startClusterManagerOnlyNode(clusterSettings); + internalCluster().startNodes(3, clusterSettings); + ensureStableCluster(4); + + assertRemoteStoreRepositoryOnAllNodes(); + } + + public void testMultiNodeClusterOnlyDataRepositoryRegistration() { + Settings clusterSettings = remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME); + internalCluster().startNodes(3, clusterSettings); + ensureStableCluster(3); + + assertRemoteStoreRepositoryOnAllNodes(); + } + + public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() { + internalCluster().startClusterManagerOnlyNodes(3); + internalCluster().startNodes(3); + ensureStableCluster(6); + + assertRemoteStoreRepositoryOnAllNodes(); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java index 263b13645e605..c21896f74d817 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java @@ -61,7 +61,6 @@ private Map validateSettingsAttributesNonNull(String settingsAtt .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(key))); } - // TODO: Add logic to mark these repository as System Repository once thats merged. private RepositoryMetadata buildRepositoryMetadata(String name) { String type = validateAttributeNonNull(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); Map settingsMap = validateSettingsAttributesNonNull( diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index 4254b3efd79ed..5915252052406 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -78,6 +78,11 @@ public RemoteStoreService(Supplier repositoriesService, Thr this.threadPool = threadPool; } + /** + * Performs repository verification during node startup post its creation by invoking verify method against + * repository mentioned. This verification will happen on a local node to validate if the node is able to connect + * to the repository. + */ public void verifyRepository(List repositories, DiscoveryNode localNode) { for (Repository repository : repositories) { String verificationToken = repository.startVerification(); @@ -105,6 +110,9 @@ public void verifyRepository(List repositories, DiscoveryNode localN } } + /** + * Creates a repository during a node startup. + */ public List createRepositories(RemoteStoreNode node) { List repositories = new ArrayList<>(); for (RepositoryMetadata repositoryMetadata : node.getRepositoriesMetadata().repositories()) { @@ -153,9 +161,14 @@ private ClusterState updateRepositoryMetadata(RepositoryMetadata newRepositoryMe return ClusterState.builder(currentState).metadata(mdBuilder).build(); } - public ClusterState updateClusterStateRepositoriesMetadata(RemoteStoreNode node, ClusterState currentState) { + /** + * Updates repositories metadata in the cluster state if not already present. If a repository metadata for a + * repository is already present in the cluster state and if it's different then the joining remote store node + * repository metadata an exception will be thrown and the node will not be allowed to join the cluster. + */ + public ClusterState updateClusterStateRepositoriesMetadata(RemoteStoreNode joiningNode, ClusterState currentState) { ClusterState newState = ClusterState.builder(currentState).build(); - for (RepositoryMetadata newRepositoryMetadata : node.getRepositoriesMetadata().repositories()) { + for (RepositoryMetadata newRepositoryMetadata : joiningNode.getRepositoriesMetadata().repositories()) { newState = updateRepositoryMetadata(newRepositoryMetadata, newState); } return newState; From d72aa587e70a77dc86e5b51edcc869a6f706baac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 27 Aug 2023 13:04:09 +0530 Subject: [PATCH 05/34] Using CoundDownLatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteStoreService.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index 5915252052406..05a2614843759 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -26,8 +26,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -86,23 +85,28 @@ public RemoteStoreService(Supplier repositoriesService, Thr public void verifyRepository(List repositories, DiscoveryNode localNode) { for (Repository repository : repositories) { String verificationToken = repository.startVerification(); - ExecutorService executor = threadPool.executor(ThreadPool.Names.GENERIC); - executor.execute(() -> { + String repositoryName = repository.getMetadata().name(); + CountDownLatch repositoryVerificationLatch = new CountDownLatch(1); + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { try { repository.verify(verificationToken, localNode); + logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); + repositoryVerificationLatch.countDown(); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); - throw new RepositoryVerificationException(repository.getMetadata().name(), e.getMessage()); + throw new RepositoryVerificationException(repositoryName, e.getMessage()); } }); // TODO: See if using listener here which is async makes sense, made this sync as - // we need the repository registration for remote store node to be completed before the bootstrap - // completes. + // we need the repository registration for remote store node to be completed before the bootstrap + // completes. try { - if(executor.awaitTermination(5000, TimeUnit.MILLISECONDS)) { - throw new RepositoryVerificationException(repository.getMetadata().name(), "could not complete " + - "repository verification within timeout."); + if (repositoryVerificationLatch.await(1000, TimeUnit.MILLISECONDS) == false) { + throw new RepositoryVerificationException( + repository.getMetadata().name(), + "could not complete " + "repository verification within timeout." + ); } } catch (InterruptedException e) { throw new RepositoryVerificationException(repository.getMetadata().name(), e.getMessage()); From bf5bfda7eb886c3ccab5275aa31b3ed98ef50115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 27 Aug 2023 16:55:56 +0530 Subject: [PATCH 06/34] Fixing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../cluster/remotestore/RemoteStoreNode.java | 27 +- .../cluster/coordination/Coordinator.java | 4 +- .../cluster/coordination/JoinHelper.java | 4 +- .../coordination/JoinTaskExecutor.java | 11 +- .../cluster/coordination/JoinHelperTests.java | 21 + .../coordination/JoinTaskExecutorTests.java | 493 +++++++++++++++++- .../cluster/coordination/NodeJoinTests.java | 6 +- .../discovery/DiscoveryModuleTests.java | 7 +- .../snapshots/SnapshotResiliencyTests.java | 6 +- .../AbstractCoordinatorTestCase.java | 17 +- 10 files changed, 570 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java index c21896f74d817..1176a7fcd2c81 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java @@ -47,25 +47,36 @@ public RemoteStoreNode(DiscoveryNode node) { private String validateAttributeNonNull(String attributeKey) { String attributeValue = node.getAttributes().get(attributeKey); if (attributeValue == null || attributeValue.isEmpty()) { - throw new IllegalStateException("joining node [" + this + "] doesn't have the node attribute [" + attributeKey + "]."); + throw new IllegalStateException("joining node [" + this.node + "] doesn't have the node attribute [" + attributeKey + "]"); } return attributeValue; } - private Map validateSettingsAttributesNonNull(String settingsAttributeKeyPrefix) { - return node.getAttributes() + private Map validateSettingsAttributesNonNull(String repositoryName) { + String settingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + repositoryName + ); + Map settingsMap = node.getAttributes() .keySet() .stream() .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(key))); + + if (settingsMap.isEmpty()) { + throw new IllegalStateException( + "joining node [" + this.node + "] doesn't have settings attribute for [" + repositoryName + "] repository" + ); + } + + return settingsMap; } private RepositoryMetadata buildRepositoryMetadata(String name) { String type = validateAttributeNonNull(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); - Map settingsMap = validateSettingsAttributesNonNull( - String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, name) - ); + Map settingsMap = validateSettingsAttributesNonNull(name); Settings.Builder settings = Settings.builder(); settingsMap.forEach(settings::put); @@ -74,8 +85,8 @@ private RepositoryMetadata buildRepositoryMetadata(String name) { } private RepositoriesMetadata buildRepositoriesMetadata() { - String segmentRepositoryName = node.getAttributes().get(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); - String translogRepositoryName = node.getAttributes().get(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + String segmentRepositoryName = validateAttributeNonNull(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + String translogRepositoryName = validateAttributeNonNull(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); if (segmentRepositoryName.equals(translogRepositoryName)) { return new RepositoriesMetadata(Collections.singletonList(buildRepositoryMetadata(segmentRepositoryName))); } else { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 1a2ec948d0de9..01473bc69cae0 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -220,6 +220,7 @@ public Coordinator( allocationService, clusterManagerService, transportService, + remoteStoreService, this::getCurrentTerm, this::getStateForClusterManagerService, this::handleJoinRequest, @@ -228,8 +229,7 @@ public Coordinator( rerouteService, nodeHealthService, this::onNodeCommissionStatusChange, - namedWriteableRegistry, - remoteStoreService + namedWriteableRegistry ); this.persistedStateSupplier = persistedStateSupplier; this.noClusterManagerBlockService = new NoClusterManagerBlockService(settings, clusterSettings); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 5f8b332525cb1..50a824f5ae300 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -136,6 +136,7 @@ public class JoinHelper { AllocationService allocationService, ClusterManagerService clusterManagerService, TransportService transportService, + RemoteStoreService remoteStoreService, LongSupplier currentTermSupplier, Supplier currentStateSupplier, BiConsumer joinHandler, @@ -144,8 +145,7 @@ public class JoinHelper { RerouteService rerouteService, NodeHealthService nodeHealthService, Consumer nodeCommissioned, - NamedWriteableRegistry namedWriteableRegistry, - RemoteStoreService remoteStoreService + NamedWriteableRegistry namedWriteableRegistry ) { this.clusterManagerService = clusterManagerService; this.transportService = transportService; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index da1c2c03cd342..ee163566fe40d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -458,9 +458,8 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) * A remote store node is the one which holds all the remote store attributes and a remote store cluster is * the one which has only homogeneous remote store nodes with same node attributes * - * TODO: When we support migration from remote store cluster to non remote store and vice versa the migration - * setting {@link RemoteStoreService::REMOTE_STORE_MIGRATION_SETTING} will be help determine if a non - * remote store node is allowed to join the remote store cluster and vice versa. + * TODO: When we support moving from remote store cluster to non remote store and vice versa the this logic will + * needs to be modified. */ public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, ClusterState currentState) { List existingNodes = new ArrayList<>(currentState.getNodes().getNodes().values()); @@ -487,18 +486,18 @@ public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode + "] is trying to join a remote store cluster with incompatible node attributes in " + "comparison with existing node [" + existingNode - + "]." + + "]" ); } } else { throw new IllegalStateException( - "a remote store node [" + joiningNode + "] is trying to join a non remote store cluster." + "a remote store node [" + joiningNode + "] is trying to join a non remote store cluster" ); } } else { if (existingNode.isRemoteStoreNode()) { throw new IllegalStateException( - "a non remote store node [" + joiningNode + "] is trying to join a remote store cluster." + "a non remote store node [" + joiningNode + "] is trying to join a remote store cluster" ); } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java index be0161b84d6fa..d3fbb0cfce09d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java @@ -34,22 +34,27 @@ import org.apache.logging.log4j.Level; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.NotClusterManagerException; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.SetOnce; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.transport.TransportResponse; import org.opensearch.monitor.StatusInfo; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; import org.opensearch.test.transport.CapturingTransport.CapturedRequest; import org.opensearch.test.transport.MockTransport; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.BytesTransportRequest; import org.opensearch.transport.RemoteTransportException; import org.opensearch.transport.TransportException; @@ -73,6 +78,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.Is.is; +import static org.mockito.Mockito.mock; public class JoinHelperTests extends OpenSearchTestCase { private final NamedWriteableRegistry namedWriteableRegistry = DEFAULT_NAMED_WRITABLE_REGISTRY; @@ -97,6 +103,7 @@ public void testJoinDeduplication() { null, null, transportService, + buildRemoteStoreService(transportService, deterministicTaskQueue.getThreadPool()), () -> 0L, () -> null, (joinRequest, joinCallback) -> { @@ -282,6 +289,7 @@ public void testJoinFailureOnUnhealthyNodes() { null, null, transportService, + buildRemoteStoreService(transportService, deterministicTaskQueue.getThreadPool()), () -> 0L, () -> null, (joinRequest, joinCallback) -> { @@ -481,6 +489,7 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin null, null, transportService, + buildRemoteStoreService(transportService, deterministicTaskQueue.getThreadPool()), () -> 0L, () -> localClusterState, (joinRequest, joinCallback) -> { @@ -500,6 +509,18 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin return new TestClusterSetup(deterministicTaskQueue, localNode, transportService, localClusterState, joinHelper, capturingTransport); } + private RemoteStoreService buildRemoteStoreService(TransportService transportService, ThreadPool threadPool) { + RepositoriesService repositoriesService = new RepositoriesService( + Settings.EMPTY, + mock(ClusterService.class), + transportService, + Collections.emptyMap(), + Collections.emptyMap(), + threadPool + ); + return new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + } + private static class TestClusterSetup { public final DeterministicTaskQueue deterministicTaskQueue; public final DiscoveryNode localNode; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index bd45985379e7d..a886bf172c508 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -33,6 +33,7 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; @@ -42,21 +43,33 @@ import org.opensearch.cluster.decommission.NodeDecommissionedException; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.VersionUtils.allVersions; import static org.opensearch.test.VersionUtils.maxCompatibleVersion; import static org.opensearch.test.VersionUtils.randomCompatibleVersion; @@ -174,8 +187,15 @@ public void testUpdatesNodeWithNewRoles() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - - final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, rerouteService); + final RemoteStoreService remoteStoreService = mock(RemoteStoreService.class); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreService + ); final DiscoveryNode clusterManagerNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); @@ -271,8 +291,15 @@ public void testJoinFailedForDecommissionedNode() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - - final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, rerouteService); + final RemoteStoreService remoteStoreService = mock(RemoteStoreService.class); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreService + ); final DiscoveryNode clusterManagerNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); @@ -327,6 +354,383 @@ public void testJoinClusterWithDecommissionFailed() { JoinTaskExecutor.ensureNodeCommissioned(discoveryNode, metadata); } + public void testJoinClusterWithNonRemoteStoreNodeJoining() { + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).build(); + + DiscoveryNode joiningNode = newDiscoveryNode(Collections.emptyMap()); + JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + } + + public void testJoinClusterWithRemoteStoreNodeJoining() { + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + } + + public void testJoinClusterWithNonRemoteStoreNodeJoiningNonRemoteStoreCluster() { + final DiscoveryNode existingNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(Collections.emptyMap()); + + JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + } + + public void testPreventJoinClusterWithRemoteStoreNodeJoiningNonRemoteStoreCluster() { + final DiscoveryNode existingNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + Exception e = assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + ); + assertTrue(e.getMessage().equals("a remote store node [" + joiningNode + "] is trying to join a non remote " + "store cluster")); + } + + public void testJoinClusterWithRemoteStoreNodeJoiningRemoteStoreCluster() { + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + } + + public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentAttributesJoiningRemoteStoreCluster() { + Map existingNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + existingNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { + if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY + && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY) { + remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue() + "-new"); + validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue()); + } + } + } + + public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentNameAttributesJoiningRemoteStoreCluster() { + Map existingNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + existingNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { + if (REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO + "new", TRANSLOG_REPO); + validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + } else if (REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO + "new"); + validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + } + } + } + + public void testPreventJoinClusterWithNonRemoteStoreNodeJoiningRemoteStoreCluster() { + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO), + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + DiscoveryNode joiningNode = newDiscoveryNode(Collections.emptyMap()); + Exception e = assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + ); + assertTrue(e.getMessage().equals("a non remote store node [" + joiningNode + "] is trying to join a remote " + "store cluster")); + } + + public void testPreventJoinClusterWithRemoteStoreNodeWithPartialAttributesJoiningRemoteStoreCluster() { + Map existingNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + final DiscoveryNode existingNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + existingNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(existingNode).localNodeId(existingNode.getId()).build()) + .build(); + + for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { + remoteStoreNodeAttributes.put(nodeAttribute.getKey(), null); + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); + Exception e = assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + ); + assertTrue( + e.getMessage().equals("joining node [" + joiningNode + "] doesn't have the node attribute [" + nodeAttribute.getKey() + "]") + ); + + remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue()); + } + } + + public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception { + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + final AllocationService allocationService = mock(AllocationService.class); + when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); + final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); + final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreService + ); + + final DiscoveryNode clusterManagerNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(clusterManagerNode) + .localNodeId(clusterManagerNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + ) + .build(); + + final ClusterStateTaskExecutor.ClusterTasksResult result = joinTaskExecutor.execute( + clusterState, + List.of(new JoinTaskExecutor.Task(clusterManagerNode, "elect leader")) + ); + assertThat(result.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); + assertTrue(taskResult.isSuccess()); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2); + } + + public void testUpdatesClusterStateWithMultiNodeCluster() throws Exception { + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO); + final AllocationService allocationService = mock(AllocationService.class); + when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); + final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); + final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreService + ); + + final DiscoveryNode clusterManagerNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final RepositoryMetadata segmentRepositoryMetadata = buildRepositoryMetadata(clusterManagerNode, SEGMENT_REPO); + final RepositoryMetadata translogRepositoryMetadata = buildRepositoryMetadata(clusterManagerNode, TRANSLOG_REPO); + List repositoriesMetadata = new ArrayList<>() { + { + add(segmentRepositoryMetadata); + add(translogRepositoryMetadata); + } + }; + + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(clusterManagerNode) + .localNodeId(clusterManagerNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + ) + .metadata(Metadata.builder().putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(repositoriesMetadata))) + .build(); + + final DiscoveryNode joiningNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterStateTaskExecutor.ClusterTasksResult result = joinTaskExecutor.execute( + clusterState, + List.of(new JoinTaskExecutor.Task(joiningNode, "test")) + ); + assertThat(result.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); + assertTrue(taskResult.isSuccess()); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2); + } + + public void testUpdatesClusterStateWithSingleNodeClusterAndSameRepository() throws Exception { + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(COMMON_REPO, COMMON_REPO); + final AllocationService allocationService = mock(AllocationService.class); + when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); + final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); + final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreService + ); + + final DiscoveryNode clusterManagerNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(clusterManagerNode) + .localNodeId(clusterManagerNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + ) + .build(); + + final ClusterStateTaskExecutor.ClusterTasksResult result = joinTaskExecutor.execute( + clusterState, + List.of(new JoinTaskExecutor.Task(clusterManagerNode, "elect leader")) + ); + assertThat(result.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); + assertTrue(taskResult.isSuccess()); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 1); + } + + public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throws Exception { + Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(COMMON_REPO, COMMON_REPO); + final AllocationService allocationService = mock(AllocationService.class); + when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); + final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); + final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + + final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( + Settings.EMPTY, + allocationService, + logger, + rerouteService, + remoteStoreService + ); + + final DiscoveryNode clusterManagerNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final RepositoryMetadata segmentRepositoryMetadata = buildRepositoryMetadata(clusterManagerNode, COMMON_REPO); + List repositoriesMetadata = new ArrayList<>() { + { + add(segmentRepositoryMetadata); + } + }; + + final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) + .nodes( + DiscoveryNodes.builder() + .add(clusterManagerNode) + .localNodeId(clusterManagerNode.getId()) + .clusterManagerNodeId(clusterManagerNode.getId()) + ) + .metadata(Metadata.builder().putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(repositoriesMetadata))) + .build(); + + final DiscoveryNode joiningNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + remoteStoreNodeAttributes, + DiscoveryNodeRole.BUILT_IN_ROLES, + Version.CURRENT + ); + + final ClusterStateTaskExecutor.ClusterTasksResult result = joinTaskExecutor.execute( + clusterState, + List.of(new JoinTaskExecutor.Task(joiningNode, "test")) + ); + assertThat(result.executionResults.entrySet(), hasSize(1)); + final ClusterStateTaskExecutor.TaskResult taskResult = result.executionResults.values().iterator().next(); + assertTrue(taskResult.isSuccess()); + validateRepositoryMetadata(result.resultingState, clusterManagerNode, 1); + } + + private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode existingNode, int expectedRepositories) + throws Exception { + + final RepositoriesMetadata repositoriesMetadata = updatedState.metadata().custom(RepositoriesMetadata.TYPE); + assertTrue(repositoriesMetadata.repositories().size() == expectedRepositories); + if (repositoriesMetadata.repositories().size() == 2) { + final RepositoryMetadata segmentRepositoryMetadata = buildRepositoryMetadata(existingNode, SEGMENT_REPO); + final RepositoryMetadata translogRepositoryMetadata = buildRepositoryMetadata(existingNode, TRANSLOG_REPO); + for (RepositoryMetadata repositoryMetadata : repositoriesMetadata.repositories()) { + if (repositoryMetadata.name().equals(segmentRepositoryMetadata.name())) { + assertTrue(segmentRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } else if (repositoryMetadata.name().equals(segmentRepositoryMetadata.name())) { + assertTrue(translogRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)); + } + } + } else if (repositoriesMetadata.repositories().size() == 1) { + final RepositoryMetadata repositoryMetadata = buildRepositoryMetadata(existingNode, COMMON_REPO); + assertTrue(repositoryMetadata.equalsIgnoreGenerations(repositoriesMetadata.repositories().get(0))); + } else { + throw new Exception("Stack overflow example: checkedExceptionThrower"); + } + } + private DiscoveryNode newDiscoveryNode(Map attributes) { return new DiscoveryNode( randomAlphaOfLength(10), @@ -337,4 +741,85 @@ private DiscoveryNode newDiscoveryNode(Map attributes) { Version.CURRENT ); } + + private static final String SEGMENT_REPO = "segment-repo"; + private static final String TRANSLOG_REPO = "translog-repo"; + private static final String COMMON_REPO = "remote-repo"; + + private Map remoteStoreNodeAttributes(String segmentRepoName, String translogRepoName) { + String segmentRepositoryTypeAttributeKey = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + segmentRepoName + ); + String segmentRepositorySettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + segmentRepoName + ); + String translogRepositoryTypeAttributeKey = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + translogRepoName + ); + String translogRepositorySettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + translogRepoName + ); + + return new HashMap<>() { + { + put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName); + put(segmentRepositoryTypeAttributeKey, "s3"); + put(segmentRepositorySettingsAttributeKeyPrefix + "bucket", "segment_bucket"); + put(segmentRepositorySettingsAttributeKeyPrefix + "base_path", "/segment/path"); + put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName); + putIfAbsent(translogRepositoryTypeAttributeKey, "s3"); + putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "bucket", "translog_bucket"); + putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "base_path", "/translog/path"); + } + }; + } + + private void validateAttributes( + Map remoteStoreNodeAttributes, + Map.Entry existingNodeAttribute, + ClusterState currentState, + DiscoveryNode existingNode + ) { + DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); + Exception e = assertThrows( + IllegalStateException.class, + () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + ); + assertTrue( + e.getMessage() + .equals( + "a remote store node [" + + joiningNode + + "] is trying to join a remote store cluster with incompatible node attributes in " + + "comparison with existing node [" + + existingNode + + "]" + ) + ); + } + + private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + Map nodeAttributes = node.getAttributes(); + String type = nodeAttributes.get(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); + + String settingsAttributeKeyPrefix = String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, name); + Map settingsMap = node.getAttributes() + .keySet() + .stream() + .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> node.getAttributes().get(key))); + + Settings.Builder settings = Settings.builder(); + settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + + return new RepositoryMetadata(name, type, settings.build()); + } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java index 8d798b38dc023..fa4bbcc934906 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java @@ -33,6 +33,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.OpenSearchAllocationTestCase; @@ -90,6 +91,8 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import org.mockito.Mockito; + import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; @@ -264,7 +267,8 @@ protected void onSendRequest( (s, p, r) -> {}, ElectionStrategy.DEFAULT_INSTANCE, nodeHealthService, - persistedStateRegistry + persistedStateRegistry, + Mockito.mock(RemoteStoreService.class) ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index b32dd7c6c240b..2e230ca5514cc 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -32,6 +32,7 @@ package org.opensearch.discovery; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.coordination.PersistedStateRegistry; @@ -77,6 +78,8 @@ public class DiscoveryModuleTests extends OpenSearchTestCase { private ClusterSettings clusterSettings; private GatewayMetaState gatewayMetaState; + private RemoteStoreService remoteStoreService; + public interface DummyHostsProviderPlugin extends DiscoveryPlugin { Map> impl(); @@ -99,6 +102,7 @@ public void setupDummyServices() { clusterApplier = mock(ClusterApplier.class); clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); gatewayMetaState = mock(GatewayMetaState.class); + remoteStoreService = mock(RemoteStoreService.class); } @After @@ -122,7 +126,8 @@ private DiscoveryModule newModule(Settings settings, List plugi gatewayMetaState, mock(RerouteService.class), null, - new PersistedStateRegistry() + new PersistedStateRegistry(), + remoteStoreService ); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 37ea8a28bd4b4..75d2602609e4b 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -40,6 +40,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.RequestValidators; import org.opensearch.action.StepListener; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -1892,6 +1893,7 @@ private final class TestClusterNode { private final ClusterInfoService clusterInfoService; private Coordinator coordinator; + private RemoteStoreService remoteStoreService; private Map actions = new HashMap<>(); @@ -1997,6 +1999,7 @@ public void onFailure(final Exception e) { emptyMap(), threadPool ); + remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService( settings, @@ -2513,7 +2516,8 @@ public void start(ClusterState initialState) { rerouteService, ElectionStrategy.DEFAULT_INSTANCE, () -> new StatusInfo(HEALTHY, "healthy-info"), - persistedStateRegistry + persistedStateRegistry, + remoteStoreService ); clusterManagerService.setClusterStatePublisher(coordinator); coordinator.start(); diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index c6960b5ca33ff..347dd3cdfab4f 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -39,6 +39,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskListener; @@ -59,6 +60,7 @@ import org.opensearch.cluster.service.FakeThreadPoolClusterManagerService; import org.opensearch.common.Nullable; import org.opensearch.common.Randomness; +import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -85,6 +87,7 @@ import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.disruption.DisruptableMockTransport; import org.opensearch.test.disruption.DisruptableMockTransport.ConnectionStatus; @@ -1035,6 +1038,8 @@ class ClusterNode { TransportService transportService; private DisruptableMockTransport mockTransport; private NodeHealthService nodeHealthService; + private RepositoriesService repositoriesService; + private RemoteStoreService remoteStoreService; List> extraJoinValidators = new ArrayList<>(); ClusterNode(int nodeIndex, boolean clusterManagerEligible, Settings nodeSettings, NodeHealthService nodeHealthService) { @@ -1129,6 +1134,15 @@ protected Optional getDisruptableMockTransport(Transpo clusterService.setNodeConnectionsService( new NodeConnectionsService(clusterService.getSettings(), threadPool, transportService) ); + repositoriesService = new RepositoriesService( + settings, + clusterService, + transportService, + Collections.emptyMap(), + Collections.emptyMap(), + threadPool + ); + remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); final Collection> onJoinValidators = Collections.singletonList( (dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)) ); @@ -1149,7 +1163,8 @@ protected Optional getDisruptableMockTransport(Transpo (s, p, r) -> {}, getElectionStrategy(), nodeHealthService, - persistedStateRegistry + persistedStateRegistry, + remoteStoreService ); clusterManagerService.setClusterStatePublisher(coordinator); final GatewayService gatewayService = new GatewayService( From f2251227e30e022f2fd7050e39556ed14e051919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 27 Aug 2023 17:11:34 +0530 Subject: [PATCH 07/34] Updating CompatibilityMode enum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../admin/cluster/remotestore/RemoteStoreService.java | 6 +++--- .../opensearch/cluster/coordination/JoinTaskExecutor.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index 05a2614843759..2e522e17613c6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -40,15 +40,15 @@ public class RemoteStoreService { private final ThreadPool threadPool; public static final Setting REMOTE_STORE_COMPATIBILITY_MODE_SETTING = Setting.simpleString( "remote_store.compatibility_mode", - CompatibilityMode.ALLOW_ONLY_REMOTE_STORE_NODES.value, + CompatibilityMode.STRICT.value, CompatibilityMode::validate, Setting.Property.Dynamic, Setting.Property.NodeScope ); public enum CompatibilityMode { - ALLOW_ONLY_REMOTE_STORE_NODES("allow_only_remote_store_nodes"), - ALLOW_ALL_NODES("allow_all_nodes"); + STRICT("strict"), + ALLOW_MIX("allow_mix"); public static CompatibilityMode validate(String compatibilityMode) { try { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index ee163566fe40d..7680555ae2379 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -61,7 +61,7 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode.ALLOW_ONLY_REMOTE_STORE_NODES; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode.STRICT; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -473,7 +473,7 @@ public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode // TODO: The below check is valid till we support migration, once we start supporting migration a remote // store node will be able to join a non remote store cluster and vice versa. #7986 String remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(currentState.metadata().settings()); - if (ALLOW_ONLY_REMOTE_STORE_NODES.value.equals(remoteStoreCompatibilityMode)) { + if (STRICT.value.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { From 64d0e31e5591981b616a38206ec7aa4220b7ae2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 27 Aug 2023 18:10:22 +0530 Subject: [PATCH 08/34] Updating REMOTE_STORE_COMPATIBILITY_MODE_SETTING value enum javadoc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../action/admin/cluster/remotestore/RemoteStoreService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index 2e522e17613c6..35bc66f8f8d44 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -46,6 +46,11 @@ public class RemoteStoreService { Setting.Property.NodeScope ); + /** + * Node join compatibility mode introduced with remote store. + * + * @opensearch.internal + */ public enum CompatibilityMode { STRICT("strict"), ALLOW_MIX("allow_mix"); From 6284bb36474e6a7e553ae5f82380a5289d9c5d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 28 Aug 2023 14:33:41 +0530 Subject: [PATCH 09/34] Addressed few comments, updated javadocs and added verify logic to be updated once system repo pr is merged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteStoreService.java | 29 ++++++++++++++----- .../cluster/remotestore/package-info.java | 10 +++++++ .../coordination/JoinTaskExecutor.java | 21 +++++++++----- .../blobstore/BlobStoreRepository.java | 6 ++++ 4 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/cluster/remotestore/package-info.java diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index 35bc66f8f8d44..d69f3ffc39188 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -31,7 +31,7 @@ import java.util.function.Supplier; /** - * Contains all the method needed for a remote store node lifecycle. + * Contains all the method needed for a remote store backed node lifecycle. */ public class RemoteStoreService { @@ -47,7 +47,7 @@ public class RemoteStoreService { ); /** - * Node join compatibility mode introduced with remote store. + * Node join compatibility mode introduced with remote backed storage. * * @opensearch.internal */ @@ -88,6 +88,21 @@ public RemoteStoreService(Supplier repositoriesService, Thr * to the repository. */ public void verifyRepository(List repositories, DiscoveryNode localNode) { + /* + for (Repository repository : repositories) { + String verificationToken = repository.startVerification(); + String repositoryName = repository.getMetadata().name(); + try { + repository.verify(verificationToken, localNode); + logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); + throw new RepositoryVerificationException(repositoryName, e.getMessage()); + } + } + Replace the below code with this once #9088 is merged. + */ + for (Repository repository : repositories) { String verificationToken = repository.startVerification(); String repositoryName = repository.getMetadata().name(); @@ -104,8 +119,8 @@ public void verifyRepository(List repositories, DiscoveryNode localN }); // TODO: See if using listener here which is async makes sense, made this sync as - // we need the repository registration for remote store node to be completed before the bootstrap - // completes. + // we need the repository registration for remote store backed node to be completed before the + // bootstrap completes. try { if (repositoryVerificationLatch.await(1000, TimeUnit.MILLISECONDS) == false) { throw new RepositoryVerificationException( @@ -128,7 +143,7 @@ public List createRepositories(RemoteStoreNode node) { RepositoriesService.validate(repositoryMetadata.name()); Repository repository = repositoriesService.get().createRepository(repositoryMetadata); logger.info( - "Remote store repository with name {} and type {} created.", + "remote backed storage repository with name {} and type {} created.", repository.getMetadata().name(), repository.getMetadata().type() ); @@ -172,8 +187,8 @@ private ClusterState updateRepositoryMetadata(RepositoryMetadata newRepositoryMe /** * Updates repositories metadata in the cluster state if not already present. If a repository metadata for a - * repository is already present in the cluster state and if it's different then the joining remote store node - * repository metadata an exception will be thrown and the node will not be allowed to join the cluster. + * repository is already present in the cluster state and if it's different then the joining remote store backed + * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ public ClusterState updateClusterStateRepositoriesMetadata(RemoteStoreNode joiningNode, ClusterState currentState) { ClusterState newState = ClusterState.builder(currentState).build(); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/package-info.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/package-info.java new file mode 100644 index 0000000000000..adb42ffa4032f --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Restore remote store transport handler. */ +package org.opensearch.action.admin.cluster.remotestore; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 7680555ae2379..11830772d662b 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -189,9 +189,9 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } else if (currentNodes.nodeExistsWithSameRoles(node)) { logger.debug("received a join request for an existing node [{}]", node); if (node.isRemoteStoreNode()) { - /** joinCluster for remote store node is invoked here as elect leader task can have same node - * present in join task as well as current node. We want the repositories to be registered during - * first node join. See + /** cluster state is updated here as elect leader task can have same node present in join task as + * well as current node. We want the repositories to be added in cluster state during first node + * join. See * {@link org.opensearch.gateway.GatewayMetaState#prepareInitialClusterState(TransportService, ClusterService, ClusterState)} **/ newState = ClusterState.builder( remoteStoreService.updateClusterStateRepositoriesMetadata(new RemoteStoreNode(node), currentState) @@ -219,6 +219,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo joiniedNodeNameIds.put(node.getName(), node.getId()); } if (node.isRemoteStoreNode()) { + // Try updating repositories metadata in cluster state once its compatible with the cluster. newState = ClusterState.builder( remoteStoreService.updateClusterStateRepositoriesMetadata(new RemoteStoreNode(node), currentState) ); @@ -452,11 +453,15 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) } /** - * The method ensures two conditions - - * 1. The joining node is remote store if it is joining a remote store cluster. - * 2. The joining node is non-remote store if it is joining a non-remote store cluster. - * A remote store node is the one which holds all the remote store attributes and a remote store cluster is - * the one which has only homogeneous remote store nodes with same node attributes + * The method ensures homogeneity - + * 1. The joining node has to be a remote store backed if it's joining a remote store backed cluster. Validates + * remote store attributes of joining node against the existing nodes of cluster. + * 2. The joining node has to be a non-remote store backed if it is joining a non-remote store backed cluster. + * Validates no remote store attributes are present in joining node as existing nodes in the cluster doesn't have + * remote store attributes. + * + * A remote store backed node is the one which holds all the remote store attributes and a remote store backed + * cluster is the one which has only homogeneous remote store backed nodes with same node attributes * * TODO: When we support moving from remote store cluster to non remote store and vice versa the this logic will * needs to be modified. diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index b850c7555c174..729c7669bc907 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -3131,6 +3131,12 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In @Override public void verify(String seed, DiscoveryNode localNode) { + /* + if(!isSystemRepository) { + assertSnapshotOrGenericThread(); + } + Update the assertion method invocation with this once #9088 is merged. + */ assertSnapshotOrGenericThread(); if (isReadOnly()) { try { From edd214872f815f7d2f38a1b52846a287dfbd90b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 28 Aug 2023 19:42:26 +0530 Subject: [PATCH 10/34] Introducing verifylocally to perform verification without cluster being formed such that a node is able to connect to connect and has necessary permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../cluster/remotestore/RemoteStoreNode.java | 24 +++--- .../remotestore/RemoteStoreService.java | 48 +----------- .../cluster/node/DiscoveryNode.java | 2 +- .../repositories/FilterRepository.java | 5 ++ .../opensearch/repositories/Repository.java | 7 ++ .../blobstore/BlobStoreRepository.java | 75 +++++++++++++++++-- .../index/shard/RestoreOnlyRepository.java | 3 + 7 files changed, 101 insertions(+), 63 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java index 1176a7fcd2c81..1b49ccb06f917 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java @@ -14,11 +14,12 @@ import org.opensearch.common.settings.Settings; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; /** @@ -85,19 +86,20 @@ private RepositoryMetadata buildRepositoryMetadata(String name) { } private RepositoriesMetadata buildRepositoriesMetadata() { - String segmentRepositoryName = validateAttributeNonNull(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); - String translogRepositoryName = validateAttributeNonNull(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); - if (segmentRepositoryName.equals(translogRepositoryName)) { - return new RepositoriesMetadata(Collections.singletonList(buildRepositoryMetadata(segmentRepositoryName))); - } else { - List repositoryMetadataList = new ArrayList<>(); - repositoryMetadataList.add(buildRepositoryMetadata(segmentRepositoryName)); - repositoryMetadataList.add(buildRepositoryMetadata(translogRepositoryName)); - return new RepositoriesMetadata(repositoryMetadataList); + List repositoryMetadataList = new ArrayList<>(); + Set repositoryNames = new HashSet<>(); + + repositoryNames.add(validateAttributeNonNull(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); + repositoryNames.add(validateAttributeNonNull(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); + + for (String repositoryName : repositoryNames) { + repositoryMetadataList.add(buildRepositoryMetadata(repositoryName)); } + + return new RepositoriesMetadata(repositoryMetadataList); } - RepositoriesMetadata getRepositoriesMetadata() { + public RepositoriesMetadata getRepositoriesMetadata() { return this.repositoriesMetadata; } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index d69f3ffc39188..0fef8389063a1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -19,15 +19,12 @@ import org.opensearch.common.settings.Setting; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; -import org.opensearch.repositories.RepositoryVerificationException; import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; /** @@ -87,50 +84,11 @@ public RemoteStoreService(Supplier repositoriesService, Thr * repository mentioned. This verification will happen on a local node to validate if the node is able to connect * to the repository. */ - public void verifyRepository(List repositories, DiscoveryNode localNode) { - /* + public void verifyRepositoriesLocally(List repositories, DiscoveryNode localNode) { for (Repository repository : repositories) { - String verificationToken = repository.startVerification(); String repositoryName = repository.getMetadata().name(); - try { - repository.verify(verificationToken, localNode); - logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); - } catch (Exception e) { - logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); - throw new RepositoryVerificationException(repositoryName, e.getMessage()); - } - } - Replace the below code with this once #9088 is merged. - */ - - for (Repository repository : repositories) { - String verificationToken = repository.startVerification(); - String repositoryName = repository.getMetadata().name(); - CountDownLatch repositoryVerificationLatch = new CountDownLatch(1); - threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { - try { - repository.verify(verificationToken, localNode); - logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); - repositoryVerificationLatch.countDown(); - } catch (Exception e) { - logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); - throw new RepositoryVerificationException(repositoryName, e.getMessage()); - } - }); - - // TODO: See if using listener here which is async makes sense, made this sync as - // we need the repository registration for remote store backed node to be completed before the - // bootstrap completes. - try { - if (repositoryVerificationLatch.await(1000, TimeUnit.MILLISECONDS) == false) { - throw new RepositoryVerificationException( - repository.getMetadata().name(), - "could not complete " + "repository verification within timeout." - ); - } - } catch (InterruptedException e) { - throw new RepositoryVerificationException(repository.getMetadata().name(), e.getMessage()); - } + repository.verifyLocally(localNode); + logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 0fb8a54209a1b..30f5be2730ac9 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -305,7 +305,7 @@ public static DiscoveryNode createLocal( ); RemoteStoreNode remoteStoreNode = new RemoteStoreNode(discoveryNode); List repositories = remoteStoreService.createRepositories(remoteStoreNode); - remoteStoreService.verifyRepository(repositories, discoveryNode); + remoteStoreService.verifyRepositoriesLocally(repositories, discoveryNode); return discoveryNode; } diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java index 08f8bcb467d03..21904d9799476 100644 --- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java @@ -162,6 +162,11 @@ public void verify(String verificationToken, DiscoveryNode localNode) { in.verify(verificationToken, localNode); } + @Override + public void verifyLocally(DiscoveryNode localNode) { + in.verifyLocally(localNode); + } + @Override public boolean isReadOnly() { return in.isReadOnly(); diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 76a3b65c9ea55..1e2c1e0398ba7 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -241,6 +241,13 @@ default RepositoryStats stats() { */ void verify(String verificationToken, DiscoveryNode localNode); + /** + * Verifies repository settings on local node by reading and writing files onto blobstore without the + * cluster-manager. + * @param localNode the local node information + */ + void verifyLocally(DiscoveryNode localNode); + /** * Returns true if the repository supports only read operations * @return true if the repository is read/only diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 729c7669bc907..9c15c71aed294 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -3131,12 +3131,6 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In @Override public void verify(String seed, DiscoveryNode localNode) { - /* - if(!isSystemRepository) { - assertSnapshotOrGenericThread(); - } - Update the assertion method invocation with this once #9088 is merged. - */ assertSnapshotOrGenericThread(); if (isReadOnly()) { try { @@ -3190,6 +3184,75 @@ public void verify(String seed, DiscoveryNode localNode) { } } + @Override + public void verifyLocally(DiscoveryNode localNode) { + String seed = UUIDs.randomBase64UUID(); + if (isReadOnly()) { + try { + latestIndexBlobId(); + } catch (Exception e) { + throw new RepositoryVerificationException( + metadata.name(), + "path " + basePath() + " is not accessible on node " + localNode, + e + ); + } + } else { + BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); + String blobName = "data-" + localNode.getId() + ".dat"; + + // Writing test data to the repository + try { + BytesArray bytes = new BytesArray(seed); + try (InputStream stream = bytes.streamInput()) { + testBlobContainer.writeBlob(blobName, stream, bytes.length(), true); + } + } catch (Exception exp) { + throw new RepositoryVerificationException( + metadata.name(), + "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", + exp + ); + } + + // Reading test data from the repository + try (InputStream localNodeDat = testBlobContainer.readBlob(blobName)) { + final String seedRead = Streams.readFully(localNodeDat).utf8ToString(); + if (seedRead.equals(seed) == false) { + throw new RepositoryVerificationException( + metadata.name(), + "Seed read was [" + seedRead + "] but expected seed [" + seed + "]" + ); + } + } catch (NoSuchFileException e) { + throw new RepositoryVerificationException( + metadata.name(), + "a file written to the store [" + + blobStore() + + "] cannot be accessed on the node [" + + localNode + + "]. " + + "This might indicate that the store [" + + blobStore() + + "] permissions don't allow reading files", + e + ); + } catch (Exception e) { + throw new RepositoryVerificationException(metadata.name(), "Failed to verify repository", e); + } + + // Trying to delete the repository once the write and read verification completes. We wont fail the + // verification if the detete fails. + // TODO: See if there is a better way to handle this deletion failure. + try { + final String testPrefix = testBlobPrefix(seed); + blobStore().blobContainer(basePath().add(testPrefix)).delete(); + } catch (Exception exp) { + logger.warn(() -> new ParameterizedMessage("[{}] cannot delete test data at {} {}", metadata.name(), basePath(), exp)); + } + } + } + @Override public String toString() { return "BlobStoreRepository[" + "[" + metadata.name() + "], [" + blobStore.get() + ']' + ']'; diff --git a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java index fbee13ab3b551..f012e516f7904 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java @@ -195,6 +195,9 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In @Override public void verify(String verificationToken, DiscoveryNode localNode) {} + @Override + public void verifyLocally(DiscoveryNode localNode) {} + @Override public void updateState(final ClusterState state) {} From c6a313568e08fdcf0d5c82c9faf9f90bcd99b80a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Tue, 29 Aug 2023 17:02:58 +0530 Subject: [PATCH 11/34] Updating/Fixing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../RemoteStoreBaseIntegTestCase.java | 41 +++++++++++-------- .../opensearch/remotestore/RemoteStoreIT.java | 21 ++++------ .../RemoteStoreRepositoryRegistrationIT.java | 29 ++++--------- .../remotestore/RemoteStoreService.java | 20 ++++----- .../coordination/JoinTaskExecutor.java | 12 ++++-- .../RepositoriesServiceTests.java | 3 ++ 6 files changed, 61 insertions(+), 65 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index e021226fb3d18..9f1317d98049a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -16,6 +16,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -25,6 +26,7 @@ import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -36,6 +38,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; @@ -114,13 +117,10 @@ protected boolean addMockInternalEngine() { @Override protected Settings nodeSettings(int nodeOrdinal) { - if (nodeAttributesSettings == null) { - nodeAttributesSettings = remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME); - } return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) - .put(nodeAttributesSettings) + .put(remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME)) .build(); } @@ -192,12 +192,15 @@ public static Settings remoteStoreClusterSettings(String segmentRepoName, String } public Settings remoteStoreNodeAttributes(String segmentRepoName, String translogRepoName) { + if (nodeAttributesSettings != null) { + return nodeAttributesSettings; + } absolutePath = randomRepoPath().toAbsolutePath(); absolutePath2 = randomRepoPath().toAbsolutePath(); if (segmentRepoName.equals(translogRepoName)) { absolutePath2 = absolutePath; } - return Settings.builder() + nodeAttributesSettings = Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) .put( String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, segmentRepoName), @@ -219,6 +222,7 @@ public Settings remoteStoreNodeAttributes(String segmentRepoName, String translo absolutePath2.toString() ) .build(); + return nodeAttributesSettings; } private Settings defaultIndexSettings() { @@ -260,18 +264,9 @@ protected void putRepository(Path path, String repoName) { assertAcked(clusterAdmin().preparePutRepository(repoName).setType("fs").setSettings(Settings.builder().put("location", path))); } - protected void setupRepo() { - setupRepo(true); - } - - protected void setupRepo(boolean startDedicatedClusterManager) { - if (startDedicatedClusterManager) { - internalCluster().startClusterManagerOnlyNode(); - } - absolutePath = randomRepoPath().toAbsolutePath(); - putRepository(absolutePath); - absolutePath2 = randomRepoPath().toAbsolutePath(); - putRepository(absolutePath2, REPOSITORY_2_NAME); + @Before + public void setup() throws Exception { + assertRepositoryMetadataPresentInClusterState(); } @After @@ -299,4 +294,16 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { return filesExisting.get(); } + void assertRepositoryMetadataPresentInClusterState() throws Exception { + assertBusy(() -> { + RepositoriesMetadata repositoriesMetadata = client().admin() + .cluster() + .prepareState() + .get() + .getState() + .metadata() + .custom(RepositoriesMetadata.TYPE); + assertTrue(repositoriesMetadata != null && !repositoriesMetadata.repositories().isEmpty()); + }, 30, TimeUnit.SECONDS); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index b87ccdb22d014..4d03e9e6e08ce 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -15,7 +15,10 @@ import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.routing.RecoverySource; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.BufferedAsyncIOProcessor; @@ -29,7 +32,6 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; import org.hamcrest.MatcherAssert; -import org.junit.Before; import java.nio.file.Path; import java.util.Arrays; @@ -47,7 +49,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.oneOf; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { protected final String INDEX_NAME = "remote-store-test-idx-1"; @@ -57,18 +59,13 @@ protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } - @Before - public void setup() { - setupRepo(); - } - @Override public Settings indexSettings() { return remoteStoreIndexSettings(0); } private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throws Exception { - internalCluster().startDataOnlyNodes(3); + internalCluster().startNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -128,8 +125,8 @@ public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogRefresh() throws Exc testPeerRecovery(randomIntBetween(2, 5), false); } - private void verifyRemoteStoreCleanup() throws Exception { - internalCluster().startDataOnlyNodes(3); + public void verifyRemoteStoreCleanup() throws Exception { + internalCluster().startNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); indexData(5, randomBoolean(), INDEX_NAME); @@ -155,7 +152,7 @@ public void testRemoteTranslogCleanup() throws Exception { } public void testStaleCommitDeletionWithInvokeFlush() throws Exception { - internalCluster().startDataOnlyNodes(1); + internalCluster().startNode(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); @@ -182,7 +179,7 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { } public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { - internalCluster().startDataOnlyNodes(1); + internalCluster().startNode(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, false, INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 87f1b83ae0dd1..8071c9328187f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -26,7 +26,7 @@ import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreRepositoryRegistrationIT extends RemoteStoreBaseIntegTestCase { @Override @@ -51,7 +51,7 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na return new RepositoryMetadata(name, type, settings.build()); } - private void assertRemoteStoreRepositoryOnAllNodes() { + private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) .state() .metadata() @@ -69,34 +69,19 @@ private void assertRemoteStoreRepositoryOnAllNodes() { } } - public void testSingleNodeClusterRepositoryRegistration() { - internalCluster().startClusterManagerOnlyNode(remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME)); - ensureStableCluster(1); - + public void testSingleNodeClusterRepositoryRegistration() throws Exception { + internalCluster().startNode(); assertRemoteStoreRepositoryOnAllNodes(); } - public void testMultiNodeClusterRepositoryRegistration() { - Settings clusterSettings = remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME); - internalCluster().startClusterManagerOnlyNode(clusterSettings); - internalCluster().startNodes(3, clusterSettings); - ensureStableCluster(4); - - assertRemoteStoreRepositoryOnAllNodes(); - } - - public void testMultiNodeClusterOnlyDataRepositoryRegistration() { - Settings clusterSettings = remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME); - internalCluster().startNodes(3, clusterSettings); - ensureStableCluster(3); - + public void testMultiNodeClusterRepositoryRegistration() throws Exception { + internalCluster().startNodes(3); assertRemoteStoreRepositoryOnAllNodes(); } - public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() { + public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() throws Exception { internalCluster().startClusterManagerOnlyNodes(3); internalCluster().startNodes(3); - ensureStableCluster(6); assertRemoteStoreRepositoryOnAllNodes(); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index 0fef8389063a1..9309771ed6fb9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -35,10 +35,10 @@ public class RemoteStoreService { private static final Logger logger = LogManager.getLogger(RemoteStoreService.class); private final Supplier repositoriesService; private final ThreadPool threadPool; - public static final Setting REMOTE_STORE_COMPATIBILITY_MODE_SETTING = Setting.simpleString( + public static final Setting REMOTE_STORE_COMPATIBILITY_MODE_SETTING = new Setting<>( "remote_store.compatibility_mode", - CompatibilityMode.STRICT.value, - CompatibilityMode::validate, + CompatibilityMode.STRICT.name(), + CompatibilityMode::parseString, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -52,7 +52,13 @@ public enum CompatibilityMode { STRICT("strict"), ALLOW_MIX("allow_mix"); - public static CompatibilityMode validate(String compatibilityMode) { + public final String mode; + + CompatibilityMode(String mode) { + this.mode = mode; + } + + public static CompatibilityMode parseString(String compatibilityMode) { try { return CompatibilityMode.valueOf(compatibilityMode.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { @@ -66,12 +72,6 @@ public static CompatibilityMode validate(String compatibilityMode) { ); } } - - public final String value; - - CompatibilityMode(String value) { - this.value = value; - } } public RemoteStoreService(Supplier repositoriesService, ThreadPool threadPool) { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 11830772d662b..53b25970a1876 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -61,6 +61,7 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode.STRICT; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; @@ -186,8 +187,11 @@ public ClusterTasksResult execute(ClusterState currentState, List jo final DiscoveryNode node = joinTask.node(); if (joinTask.isBecomeClusterManagerTask() || joinTask.isFinishElectionTask()) { // noop - } else if (currentNodes.nodeExistsWithSameRoles(node)) { - logger.debug("received a join request for an existing node [{}]", node); + } else if (currentNodes.nodeExists(node)) { + if (currentNodes.nodeExistsWithSameRoles(node)) { + logger.debug("received a join request for an existing node [{}]", node); + } + if (node.isRemoteStoreNode()) { /** cluster state is updated here as elect leader task can have same node present in join task as * well as current node. We want the repositories to be added in cluster state during first node @@ -477,8 +481,8 @@ public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode // TODO: The below check is valid till we support migration, once we start supporting migration a remote // store node will be able to join a non remote store cluster and vice versa. #7986 - String remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(currentState.metadata().settings()); - if (STRICT.value.equals(remoteStoreCompatibilityMode)) { + CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(currentState.metadata().settings()); + if (STRICT.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index 1ab48b30af2f9..e850e10895247 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -751,6 +751,9 @@ public void verify(String verificationToken, DiscoveryNode localNode) { } + @Override + public void verifyLocally(DiscoveryNode localNode) {} + @Override public boolean isReadOnly() { return false; From efeef9e063e5b3f6cfd2b17016c0ba64a4267813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Tue, 29 Aug 2023 23:56:52 +0530 Subject: [PATCH 12/34] Fixing ITs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/PrimaryTermValidationIT.java | 16 ++--- .../RemoteSegmentStatsFromNodesStatsIT.java | 6 +- .../RemoteStoreBaseIntegTestCase.java | 72 +++++++++++-------- .../opensearch/remotestore/RemoteStoreIT.java | 17 +++-- .../RemoteStoreRepositoryRegistrationIT.java | 5 +- .../remotestore/RemoteStoreRestoreIT.java | 20 +++--- .../ReplicaToPrimaryPromotionIT.java | 10 ++- .../RemoteStoreMultipartFileCorruptionIT.java | 6 -- .../multipart/RemoteStoreMultipartIT.java | 15 ++-- 9 files changed, 83 insertions(+), 84 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index f72d107a367de..a6d8a987886af 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -35,13 +35,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) - public class PrimaryTermValidationIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; @@ -64,17 +62,11 @@ public void testPrimaryTermValidation() throws Exception { .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) .build(); internalCluster().startClusterManagerOnlyNode(clusterSettings); - - // Create repository - absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); - absolutePath2 = randomRepoPath().toAbsolutePath(); - putRepository(absolutePath2, REPOSITORY_2_NAME); - - // Start data nodes and create index internalCluster().startDataOnlyNodes(2, clusterSettings); + ensureStableCluster(3); + assertRepositoryMetadataPresentInClusterState(); + + // Create index createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java index c2e79ea2de5ef..7e273828a86ef 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java @@ -25,15 +25,15 @@ public class RemoteSegmentStatsFromNodesStatsIT extends RemoteStoreBaseIntegTest private static final int CLUSTER_MANAGER_NODE_COUNT = 3; @Before - public void setup() { + public void setup() throws Exception { setupCustomCluster(); - setupRepo(false); } - private void setupCustomCluster() { + private void setupCustomCluster() throws Exception { internalCluster().startClusterManagerOnlyNodes(CLUSTER_MANAGER_NODE_COUNT); internalCluster().startDataOnlyNodes(DATA_NODE_COUNT); ensureStableCluster(DATA_NODE_COUNT + CLUSTER_MANAGER_NODE_COUNT); + assertRepositoryMetadataPresentInClusterState(); } /** diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 9f1317d98049a..eae9b504e3d31 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -24,9 +24,9 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -61,8 +61,8 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total"; protected static final String MAX_SEQ_NO_REFRESHED_OR_FLUSHED = "max-seq-no-refreshed-or-flushed"; - protected Path absolutePath; - protected Path absolutePath2; + protected Path segmentRepoPath; + protected Path translogRepoPath; protected Settings nodeAttributesSettings; private final List documentKeys = List.of( randomAlphaOfLength(5), @@ -120,7 +120,7 @@ protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) - .put(remoteStoreNodeAttributes(REPOSITORY_NAME, REPOSITORY_2_NAME)) + .put(remoteStoreNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, REPOSITORY_2_NAME, FsRepository.TYPE)) .build(); } @@ -191,36 +191,51 @@ public static Settings remoteStoreClusterSettings(String segmentRepoName, String return settingsBuilder.build(); } - public Settings remoteStoreNodeAttributes(String segmentRepoName, String translogRepoName) { + public Settings remoteStoreNodeAttributes( + String segmentRepoName, + String segmentRepoType, + String translogRepoName, + String translogRepoType + ) { if (nodeAttributesSettings != null) { return nodeAttributesSettings; } - absolutePath = randomRepoPath().toAbsolutePath(); - absolutePath2 = randomRepoPath().toAbsolutePath(); + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + String segmentRepoKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + segmentRepoName + ); + String translogRepoKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + translogRepoName + ); + String segmentRepoSettingsPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + segmentRepoName + ); + String translogRepoSettingsPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + translogRepoName + ); + if (segmentRepoName.equals(translogRepoName)) { - absolutePath2 = absolutePath; + segmentRepoPath = translogRepoPath; + segmentRepoKey = translogRepoKey; + segmentRepoSettingsPrefix = translogRepoSettingsPrefix; } + nodeAttributesSettings = Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put( - String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, segmentRepoName), - "fs" - ) - .put( - String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, segmentRepoName) - + "location", - absolutePath.toString() - ) + .put(segmentRepoKey, segmentRepoType) + .put(segmentRepoSettingsPrefix + "location", segmentRepoPath.toString()) .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) - .put( - String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, translogRepoName), - "fs" - ) - .put( - String.format(Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName) - + "location", - absolutePath2.toString() - ) + .put(translogRepoKey, translogRepoType) + .put(translogRepoSettingsPrefix + "location", translogRepoPath.toString()) .build(); return nodeAttributesSettings; } @@ -264,11 +279,6 @@ protected void putRepository(Path path, String repoName) { assertAcked(clusterAdmin().preparePutRepository(repoName).setType("fs").setSettings(Settings.builder().put("location", path))); } - @Before - public void setup() throws Exception { - assertRepositoryMetadataPresentInClusterState(); - } - @After public void teardown() { nodeAttributesSettings = null; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 4d03e9e6e08ce..e6bcf094e56c1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -15,10 +15,7 @@ import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.RepositoriesMetadata; -import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.routing.RecoverySource; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.BufferedAsyncIOProcessor; @@ -66,6 +63,8 @@ public Settings indexSettings() { private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startNodes(3); + ensureStableCluster(3); + assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -127,6 +126,8 @@ public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogRefresh() throws Exc public void verifyRemoteStoreCleanup() throws Exception { internalCluster().startNodes(3); + ensureStableCluster(3); + assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); indexData(5, randomBoolean(), INDEX_NAME); @@ -135,7 +136,7 @@ public void verifyRemoteStoreCleanup() throws Exception { .prepareGetSettings(INDEX_NAME) .get() .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); - Path indexPath = Path.of(String.valueOf(absolutePath), indexUUID); + Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID); assertTrue(getFileCount(indexPath) > 0); assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); // Delete is async. Give time for it @@ -153,6 +154,8 @@ public void testRemoteTranslogCleanup() throws Exception { public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startNode(); + ensureStableCluster(1); + assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); @@ -161,7 +164,7 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { .prepareGetSettings(INDEX_NAME) .get() .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); - Path indexPath = Path.of(String.valueOf(absolutePath), indexUUID, "/0/segments/metadata"); + Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata"); // Delete is async. assertBusy(() -> { int actualFileCount = getFileCount(indexPath); @@ -180,6 +183,8 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startNode(); + ensureStableCluster(1); + assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, false, INDEX_NAME); @@ -188,7 +193,7 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { .prepareGetSettings(INDEX_NAME) .get() .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); - Path indexPath = Path.of(String.valueOf(absolutePath), indexUUID, "/0/segments/metadata"); + Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata"); int actualFileCount = getFileCount(indexPath); // We also allow (numberOfIterations + 1) as index creation also triggers refresh. MatcherAssert.assertThat(actualFileCount, is(oneOf(numberOfIterations - 1, numberOfIterations, numberOfIterations + 1))); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 8071c9328187f..54d8db353d90d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -52,6 +52,7 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na } private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { + assertRepositoryMetadataPresentInClusterState(); RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) .state() .metadata() @@ -71,18 +72,20 @@ private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { public void testSingleNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNode(); + ensureStableCluster(1); assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNodes(3); + ensureStableCluster(3); assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() throws Exception { internalCluster().startClusterManagerOnlyNodes(3); internalCluster().startNodes(3); - + ensureStableCluster(6); assertRemoteStoreRepositoryOnAllNodes(); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 85c2514ebf00f..623e03983f111 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -21,7 +21,6 @@ import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; -import org.junit.Before; import java.io.IOException; import java.util.Arrays; @@ -53,11 +52,6 @@ protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } - @Before - public void setup() { - setupRepo(); - } - private void restore(String... indices) { boolean restoreAllShards = randomBoolean(); if (restoreAllShards) { @@ -93,7 +87,8 @@ private void verifyRestoredData(Map indexStats, String indexName) ); } - private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { + private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) + throws Exception { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); internalCluster().startDataOnlyNodes(numDataOnlyNodes); for (String index : indices.split(",")) { @@ -101,6 +96,7 @@ private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, St ensureYellowAndNoInitializingShards(index); ensureGreen(index); } + assertRepositoryMetadataPresentInClusterState(); } /** @@ -187,7 +183,7 @@ private void restoreAndVerify(int shardCount, int replicaCount, Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); @@ -267,9 +263,9 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo } } - public void testRestoreFlowAllShardsNoRedIndex() throws InterruptedException { + public void testRestoreFlowAllShardsNoRedIndex() throws Exception { int shardCount = randomIntBetween(1, 5); - prepareCluster(0, 3, INDEX_NAME, 0, shardCount); + prepareCluster(1, 3, INDEX_NAME, 0, shardCount); indexData(randomIntBetween(2, 5), true, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); @@ -285,7 +281,7 @@ public void testRestoreFlowAllShardsNoRedIndex() throws InterruptedException { public void testRestoreFlowNoRedIndex() throws Exception { int shardCount = randomIntBetween(1, 5); - prepareCluster(0, 3, INDEX_NAME, 0, shardCount); + prepareCluster(1, 3, INDEX_NAME, 0, shardCount); Map indexStats = indexData(randomIntBetween(2, 5), true, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); @@ -474,7 +470,7 @@ public void testRateLimitedRemoteDownloads() throws Exception { ) ); int shardCount = randomIntBetween(1, 3); - prepareCluster(0, 3, INDEX_NAME, 0, shardCount); + prepareCluster(1, 3, INDEX_NAME, 0, shardCount); Map indexStats = indexData(5, false, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index abad56d892d88..edea93464e650 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -22,7 +22,6 @@ import org.opensearch.test.BackgroundIndexer; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.Before; import java.util.Locale; import java.util.concurrent.CountDownLatch; @@ -38,11 +37,6 @@ public class ReplicaToPrimaryPromotionIT extends RemoteStoreBaseIntegTestCase { private int shard_count = 5; - @Before - public void setup() { - setupRepo(); - } - @Override public Settings indexSettings() { return Settings.builder().put(super.indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shard_count).build(); @@ -51,6 +45,8 @@ public Settings indexSettings() { public void testPromoteReplicaToPrimary() throws Exception { internalCluster().startNode(); internalCluster().startNode(); + ensureStableCluster(2); + assertRepositoryMetadataPresentInClusterState(); final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); shard_count = scaledRandomIntBetween(1, 5); createIndex(indexName); @@ -126,6 +122,8 @@ public void testPromoteReplicaToPrimary() throws Exception { public void testFailoverWhileIndexing() throws Exception { internalCluster().startNode(); internalCluster().startNode(); + ensureStableCluster(2); + assertRepositoryMetadataPresentInClusterState(); final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); shard_count = scaledRandomIntBetween(1, 5); createIndex(indexName); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java index f2d2e6c04d114..7112b266840ac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartFileCorruptionIT.java @@ -16,7 +16,6 @@ import org.opensearch.plugins.Plugin; import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase; import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; -import org.junit.Before; import java.util.Collection; import java.util.stream.Collectors; @@ -31,11 +30,6 @@ protected Collection> nodePlugins() { return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); } - @Before - public void setup() { - setupRepo(); - } - protected Settings remoteStoreIndexSettings() { return Settings.builder() .put(super.indexSettings()) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java index 842a576a92a38..da8ba77436f7f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -35,16 +35,17 @@ protected Collection> nodePlugins() { } @Override - protected void putRepository(Path path) { - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME) - .setType(MockFsRepositoryPlugin.TYPE) - .setSettings(Settings.builder().put("location", path)) - ); + public Settings remoteStoreNodeAttributes( + String segmentRepoName, + String segmentRepoType, + String translogRepoName, + String translogRepoType + ) { + return super.remoteStoreNodeAttributes(segmentRepoName, MockFsRepositoryPlugin.TYPE, translogRepoName, MockFsRepositoryPlugin.TYPE); } public void testRateLimitedRemoteUploads() throws Exception { - internalCluster().startDataOnlyNodes(1); + internalCluster().startNode(); Client client = client(); logger.info("--> updating repository"); Path repositoryLocation = randomRepoPath(); From 148254430735eaa2b3bee7dc6124f186375d3199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Wed, 30 Aug 2023 14:33:36 +0530 Subject: [PATCH 13/34] Removing verifyLocally and move to existing verify methods. If the repository is a system repository only local verification will happen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../cluster/remotestore/RemoteStoreNode.java | 4 + .../remotestore/RemoteStoreService.java | 4 +- .../repositories/FilterRepository.java | 8 +- .../opensearch/repositories/Repository.java | 14 +- .../blobstore/BlobStoreRepository.java | 133 ++++++------------ .../RepositoriesServiceTests.java | 6 +- .../index/shard/RestoreOnlyRepository.java | 8 +- 7 files changed, 70 insertions(+), 107 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java index 1b49ccb06f917..a6bd099679aa4 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java @@ -12,6 +12,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.util.ArrayList; import java.util.HashSet; @@ -82,6 +83,9 @@ private RepositoryMetadata buildRepositoryMetadata(String name) { Settings.Builder settings = Settings.builder(); settingsMap.forEach(settings::put); + // Repository metadata built here will always be for a system repository. + settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); + return new RepositoryMetadata(name, type, settings.build()); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java index 9309771ed6fb9..e2cee2f2f6449 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java @@ -87,7 +87,9 @@ public RemoteStoreService(Supplier repositoriesService, Thr public void verifyRepositoriesLocally(List repositories, DiscoveryNode localNode) { for (Repository repository : repositories) { String repositoryName = repository.getMetadata().name(); - repository.verifyLocally(localNode); + String verificationToken = repository.startVerification(); + repository.verify(verificationToken, localNode); + repository.endVerification(verificationToken); logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); } } diff --git a/server/src/main/java/org/opensearch/repositories/FilterRepository.java b/server/src/main/java/org/opensearch/repositories/FilterRepository.java index 21904d9799476..697ac37c4a175 100644 --- a/server/src/main/java/org/opensearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/opensearch/repositories/FilterRepository.java @@ -163,13 +163,13 @@ public void verify(String verificationToken, DiscoveryNode localNode) { } @Override - public void verifyLocally(DiscoveryNode localNode) { - in.verifyLocally(localNode); + public boolean isReadOnly() { + return in.isReadOnly(); } @Override - public boolean isReadOnly() { - return in.isReadOnly(); + public boolean isSystemRepository() { + return in.isSystemRepository(); } @Override diff --git a/server/src/main/java/org/opensearch/repositories/Repository.java b/server/src/main/java/org/opensearch/repositories/Repository.java index 1e2c1e0398ba7..10f3dc2b6b340 100644 --- a/server/src/main/java/org/opensearch/repositories/Repository.java +++ b/server/src/main/java/org/opensearch/repositories/Repository.java @@ -241,19 +241,19 @@ default RepositoryStats stats() { */ void verify(String verificationToken, DiscoveryNode localNode); - /** - * Verifies repository settings on local node by reading and writing files onto blobstore without the - * cluster-manager. - * @param localNode the local node information - */ - void verifyLocally(DiscoveryNode localNode); - /** * Returns true if the repository supports only read operations * @return true if the repository is read/only */ boolean isReadOnly(); + /** + * Returns true if the repository is managed by the system directly and doesn't allow managing the lifetime of the + * repository through external APIs + * @return true if the repository is system managed + */ + boolean isSystemRepository(); + /** * Creates a snapshot of the shard based on the index commit point. *

diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index 9c15c71aed294..daa067f0c639d 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -286,6 +286,15 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp */ public static final Setting READONLY_SETTING = Setting.boolSetting("readonly", false, Setting.Property.NodeScope); + /*** + * Setting to set repository as system repository + */ + public static final Setting SYSTEM_REPOSITORY_SETTING = Setting.boolSetting( + "system_repository", + false, + Setting.Property.NodeScope + ); + protected final boolean supportURLRepo; private final int maxShardBlobDeleteBatch; @@ -347,6 +356,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp private final boolean readOnly; + private final boolean isSystemRepository; + private final Object lock = new Object(); private final SetOnce blobContainer = new SetOnce<>(); @@ -412,6 +423,7 @@ protected BlobStoreRepository( remoteUploadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_upload_bytes_per_sec", ByteSizeValue.ZERO); remoteDownloadRateLimiter = getRateLimiter(metadata.settings(), "max_remote_download_bytes_per_sec", ByteSizeValue.ZERO); readOnly = READONLY_SETTING.get(metadata.settings()); + isSystemRepository = SYSTEM_REPOSITORY_SETTING.get(metadata.settings()); cacheRepositoryData = CACHE_REPOSITORY_DATA.get(metadata.settings()); bufferSize = Math.toIntExact(BUFFER_SIZE_SETTING.get(metadata.settings()).getBytes()); maxShardBlobDeleteBatch = MAX_SNAPSHOT_SHARD_BLOB_DELETE_BATCH_SIZE.get(metadata.settings()); @@ -1831,8 +1843,10 @@ public String startVerification() { byte[] testBytes = Strings.toUTF8Bytes(seed); BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); BytesArray bytes = new BytesArray(testBytes); - try (InputStream stream = bytes.streamInput()) { - testContainer.writeBlobAtomic("master.dat", stream, bytes.length(), true); + if (isSystemRepository == false) { + try (InputStream stream = bytes.streamInput()) { + testContainer.writeBlobAtomic("master.dat", stream, bytes.length(), true); + } } return seed; } @@ -2141,6 +2155,11 @@ public boolean isReadOnly() { return readOnly; } + @Override + public boolean isSystemRepository() { + return isSystemRepository; + } + /** * Writing a new index generation is a three step process. * First, the {@link RepositoryMetadata} entry for this repository is set into a pending state by incrementing its @@ -3156,99 +3175,33 @@ public void verify(String seed, DiscoveryNode localNode) { exp ); } - try (InputStream masterDat = testBlobContainer.readBlob("master.dat")) { - final String seedRead = Streams.readFully(masterDat).utf8ToString(); - if (seedRead.equals(seed) == false) { - throw new RepositoryVerificationException( - metadata.name(), - "Seed read from master.dat was [" + seedRead + "] but expected seed [" + seed + "]" - ); - } - } catch (NoSuchFileException e) { - throw new RepositoryVerificationException( - metadata.name(), - "a file written by cluster-manager to the store [" - + blobStore() - + "] cannot be accessed on the node [" - + localNode - + "]. " - + "This might indicate that the store [" - + blobStore() - + "] is not shared between this node and the cluster-manager node or " - + "that permissions on the store don't allow reading files written by the cluster-manager node", - e - ); - } catch (Exception e) { - throw new RepositoryVerificationException(metadata.name(), "Failed to verify repository", e); - } - } - } - - @Override - public void verifyLocally(DiscoveryNode localNode) { - String seed = UUIDs.randomBase64UUID(); - if (isReadOnly()) { - try { - latestIndexBlobId(); - } catch (Exception e) { - throw new RepositoryVerificationException( - metadata.name(), - "path " + basePath() + " is not accessible on node " + localNode, - e - ); - } - } else { - BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); - String blobName = "data-" + localNode.getId() + ".dat"; - - // Writing test data to the repository - try { - BytesArray bytes = new BytesArray(seed); - try (InputStream stream = bytes.streamInput()) { - testBlobContainer.writeBlob(blobName, stream, bytes.length(), true); - } - } catch (Exception exp) { - throw new RepositoryVerificationException( - metadata.name(), - "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", - exp - ); - } - // Reading test data from the repository - try (InputStream localNodeDat = testBlobContainer.readBlob(blobName)) { - final String seedRead = Streams.readFully(localNodeDat).utf8ToString(); - if (seedRead.equals(seed) == false) { + if (isSystemRepository == false) { + try (InputStream masterDat = testBlobContainer.readBlob("master.dat")) { + final String seedRead = Streams.readFully(masterDat).utf8ToString(); + if (seedRead.equals(seed) == false) { + throw new RepositoryVerificationException( + metadata.name(), + "Seed read from master.dat was [" + seedRead + "] but expected seed [" + seed + "]" + ); + } + } catch (NoSuchFileException e) { throw new RepositoryVerificationException( metadata.name(), - "Seed read was [" + seedRead + "] but expected seed [" + seed + "]" + "a file written by cluster-manager to the store [" + + blobStore() + + "] cannot be accessed on the node [" + + localNode + + "]. " + + "This might indicate that the store [" + + blobStore() + + "] is not shared between this node and the cluster-manager node or " + + "that permissions on the store don't allow reading files written by the cluster-manager node", + e ); + } catch (Exception e) { + throw new RepositoryVerificationException(metadata.name(), "Failed to verify repository", e); } - } catch (NoSuchFileException e) { - throw new RepositoryVerificationException( - metadata.name(), - "a file written to the store [" - + blobStore() - + "] cannot be accessed on the node [" - + localNode - + "]. " - + "This might indicate that the store [" - + blobStore() - + "] permissions don't allow reading files", - e - ); - } catch (Exception e) { - throw new RepositoryVerificationException(metadata.name(), "Failed to verify repository", e); - } - - // Trying to delete the repository once the write and read verification completes. We wont fail the - // verification if the detete fails. - // TODO: See if there is a better way to handle this deletion failure. - try { - final String testPrefix = testBlobPrefix(seed); - blobStore().blobContainer(basePath().add(testPrefix)).delete(); - } catch (Exception exp) { - logger.warn(() -> new ParameterizedMessage("[{}] cannot delete test data at {} {}", metadata.name(), basePath(), exp)); } } } diff --git a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java index e850e10895247..b20cb323e144f 100644 --- a/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/opensearch/repositories/RepositoriesServiceTests.java @@ -752,10 +752,12 @@ public void verify(String verificationToken, DiscoveryNode localNode) { } @Override - public void verifyLocally(DiscoveryNode localNode) {} + public boolean isReadOnly() { + return false; + } @Override - public boolean isReadOnly() { + public boolean isSystemRepository() { return false; } diff --git a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java index f012e516f7904..be2f895301396 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/RestoreOnlyRepository.java @@ -173,6 +173,11 @@ public boolean isReadOnly() { return false; } + @Override + public boolean isSystemRepository() { + return false; + } + @Override public void snapshotShard( Store store, @@ -195,9 +200,6 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In @Override public void verify(String verificationToken, DiscoveryNode localNode) {} - @Override - public void verifyLocally(DiscoveryNode localNode) {} - @Override public void updateState(final ClusterState state) {} From a295af6b8a4936d64eb5f6162df241fb7546eb1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Wed, 30 Aug 2023 20:32:10 +0530 Subject: [PATCH 14/34] Fixing Additional Tests, there might still be some flaky ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteIndexRecoveryIT.java | 48 ++++++++++++------- .../RemoteStoreBaseIntegTestCase.java | 12 +---- .../remotestore/RemoteStoreForceMergeIT.java | 20 ++++---- .../RemoteStoreRepositoryRegistrationIT.java | 10 ++++ .../remotestore/RemoteStoreRestoreIT.java | 4 +- .../remotestore/RemoteStoreStatsIT.java | 21 ++++---- .../blobstore/BlobStoreRepository.java | 4 +- 7 files changed, 69 insertions(+), 50 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java index 11942711b7c22..a48ab7b828157 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -15,14 +15,18 @@ import org.opensearch.index.IndexSettings; import org.opensearch.indices.recovery.IndexRecoveryIT; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.After; -import org.junit.Before; -import java.nio.file.Path; +import java.util.Locale; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -31,11 +35,34 @@ public class RemoteIndexRecoveryIT extends IndexRecoveryIT { protected static final String REPOSITORY_NAME = "test-remote-store-repo"; - protected Path absolutePath; - @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME)) + .put(repositoryNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, randomRepoPath().toAbsolutePath().toString())) + .build(); + } + + private Settings repositoryNodeAttributes(String name, String type, String location) { + String segmentRepoNameAttributeKey = "node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; + String translogRepoNameAttributeKey = "node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; + String typeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + name + ); + String settingsAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + name + ); + return Settings.builder() + .put(segmentRepoNameAttributeKey, name) + .put(translogRepoNameAttributeKey, name) + .put(typeAttributeKey, type) + .put(settingsAttributeKey + "location", location) + .build(); } @Override @@ -47,17 +74,6 @@ protected Settings featureFlagSettings() { .build(); } - @Before - @Override - public void setUp() throws Exception { - super.setUp(); - internalCluster().startClusterManagerOnlyNode(); - absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); - } - @Override public Settings indexSettings() { return Settings.builder() diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index eae9b504e3d31..bdc2ef2dedefc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -271,16 +271,8 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel .build(); } - protected void putRepository(Path path) { - putRepository(path, REPOSITORY_NAME); - } - - protected void putRepository(Path path, String repoName) { - assertAcked(clusterAdmin().preparePutRepository(repoName).setType("fs").setSettings(Settings.builder().put("location", path))); - } - @After - public void teardown() { + public void teardown() throws Exception { nodeAttributesSettings = null; assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); @@ -304,7 +296,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { return filesExisting.get(); } - void assertRepositoryMetadataPresentInClusterState() throws Exception { + public void assertRepositoryMetadataPresentInClusterState() throws Exception { assertBusy(() -> { RepositoriesMetadata repositoriesMetadata = client().admin() .cluster() diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index abc0f35d48eab..61497ed896cf5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -16,9 +16,7 @@ import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; -import org.junit.Before; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -29,7 +27,7 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 3) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreForceMergeIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; @@ -41,11 +39,6 @@ protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } - @Before - public void setup() { - setupRepo(); - } - @Override public Settings indexSettings() { return remoteStoreIndexSettings(0); @@ -96,7 +89,10 @@ private void verifyRestoredData(Map indexStats, long deletedDocs) } private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) - throws IOException { + throws Exception { + internalCluster().startNodes(3); + ensureStableCluster(3); + assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -128,19 +124,19 @@ private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlus // values for each of the flags, number of integ tests become 16 in comparison to current 2. // We have run all the 16 tests on local and they run fine. @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9294") - public void testRestoreForceMergeSingleIteration() throws IOException { + public void testRestoreForceMergeSingleIteration() throws Exception { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(1, invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); } - public void testRestoreForceMergeMultipleIterations() throws IOException { + public void testRestoreForceMergeMultipleIterations() throws Exception { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(randomIntBetween(2, 5), invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); } - public void testRestoreForceMergeMultipleIterationsDeleteAll() throws IOException { + public void testRestoreForceMergeMultipleIterationsDeleteAll() throws Exception { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(randomIntBetween(2, 3), invokeFLush, flushAfterMerge, -1); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 54d8db353d90d..1ca6d2c23cf0d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -14,6 +14,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -47,6 +49,7 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na Settings.Builder settings = Settings.builder(); settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); return new RepositoryMetadata(name, type, settings.build()); } @@ -88,4 +91,11 @@ public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() thro ensureStableCluster(6); assertRemoteStoreRepositoryOnAllNodes(); } + + public void testMultiNodeClusterRepositoryRegistrationDifferent() throws Exception { + internalCluster().startNodes(3); + internalCluster().startNode(remoteStoreNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, REPOSITORY_NAME, FsRepository.TYPE)); + ensureStableCluster(3); + assertRemoteStoreRepositoryOnAllNodes(); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 623e03983f111..4b6c98ba60afe 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -34,7 +34,7 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThan; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreRestoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; private static final String INDEX_NAMES = "test-remote-store-1,test-remote-store-2,remote-store-test-index-1,remote-store-test-index-2"; @@ -91,12 +91,12 @@ private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, St throws Exception { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); internalCluster().startDataOnlyNodes(numDataOnlyNodes); + assertRepositoryMetadataPresentInClusterState(); for (String index : indices.split(",")) { createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); ensureYellowAndNoInitializingShards(index); ensureGreen(index); } - assertRepositoryMetadataPresentInClusterState(); } /** diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 2b9fb9c497cb6..6999b2d9a4858 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -39,17 +39,19 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 3) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; @Before - public void setup() { - setupRepo(); + public void setup() throws Exception { + internalCluster().startNodes(3); + ensureStableCluster(3); + assertRepositoryMetadataPresentInClusterState(); } - public void testStatsResponseFromAllNodes() { + public void testStatsResponseFromAllNodes() throws Exception { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. @@ -117,7 +119,7 @@ public void testStatsResponseFromAllNodes() { } } - public void testStatsResponseAllShards() { + public void testStatsResponseAllShards() throws Exception { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. @@ -174,7 +176,7 @@ public void testStatsResponseAllShards() { } - public void testStatsResponseFromLocalNode() { + public void testStatsResponseFromLocalNode() throws Exception { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. @@ -415,7 +417,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr } } - public void testStatsOnShardRelocation() { + public void testStatsOnShardRelocation() throws Exception { // Scenario: // - Create index with single primary and single replica shard // - Index documents @@ -496,7 +498,7 @@ public void testStatsOnShardUnassigned() throws IOException { indexSingleDoc(INDEX_NAME); } - public void testStatsOnRemoteStoreRestore() throws IOException { + public void testStatsOnRemoteStoreRestore() throws Exception { // Creating an index with primary shard count == total nodes in cluster and 0 replicas int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0, dataNodeCount)); @@ -511,7 +513,8 @@ public void testStatsOnRemoteStoreRestore() throws IOException { ensureRed(INDEX_NAME); // Start another data node to fulfil the previously launched capacity - internalCluster().startDataOnlyNode(); + String nodeId = internalCluster().startDataOnlyNode(); + ensureStableCluster(3); // Restore index from remote store assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index daa067f0c639d..220d468a03090 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -3150,7 +3150,9 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, In @Override public void verify(String seed, DiscoveryNode localNode) { - assertSnapshotOrGenericThread(); + if (isSystemRepository == false) { + assertSnapshotOrGenericThread(); + } if (isReadOnly()) { try { latestIndexBlobId(); From d67801b106bb9f9d64f533d989564228f9ad8aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Fri, 1 Sep 2023 00:30:35 +0530 Subject: [PATCH 15/34] Removing remote store specific cluster settings instead rely on node attributes, fixed UTs and ITs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- distribution/src/config/opensearch.yml | 4 +- ...emoteStoreMockRepositoryIntegTestCase.java | 70 ++++++++++---- ...eateRemoteIndexClusterDefaultDocRepIT.java | 4 +- .../remotestore/CreateRemoteIndexIT.java | 51 +--------- .../remotestore/PrimaryTermValidationIT.java | 14 ++- .../RemoteIndexPrimaryRelocationIT.java | 6 +- .../remotestore/RemoteIndexRecoveryIT.java | 39 ++------ .../remotestore/RemoteRestoreSnapshotIT.java | 13 +-- .../RemoteSegmentStatsFromNodesStatsIT.java | 5 +- .../RemoteStoreBaseIntegTestCase.java | 92 ++++++------------- .../remotestore/RemoteStoreForceMergeIT.java | 11 +-- .../opensearch/remotestore/RemoteStoreIT.java | 10 +- .../RemoteStoreRepositoryRegistrationIT.java | 12 --- .../remotestore/RemoteStoreRestoreIT.java | 6 +- .../remotestore/RemoteStoreStatsIT.java | 7 +- .../ReplicaToPrimaryPromotionIT.java | 4 - .../SegmentReplicationUsingRemoteStoreIT.java | 13 ++- ...tReplicationWithRemoteStorePressureIT.java | 11 ++- .../multipart/RemoteStoreMultipartIT.java | 10 -- .../opensearch/snapshots/CloneSnapshotIT.java | 30 +++--- .../snapshots/DeleteSnapshotIT.java | 48 ++++------ .../RemoteIndexSnapshotStatusApiIT.java | 25 +++-- ...rvice.java => RemoteStoreNodeService.java} | 40 ++++---- .../cluster/coordination/Coordinator.java | 6 +- .../cluster/coordination/JoinHelper.java | 4 +- .../coordination/JoinTaskExecutor.java | 20 ++-- .../metadata/MetadataCreateIndexService.java | 34 ++++--- .../cluster/node/DiscoveryNode.java | 13 +-- .../common/settings/ClusterSettings.java | 11 +-- .../opensearch/discovery/DiscoveryModule.java | 4 +- .../opensearch/indices/IndicesService.java | 41 --------- .../main/java/org/opensearch/node/Node.java | 19 ++-- .../cluster/coordination/JoinHelperTests.java | 6 +- .../coordination/JoinTaskExecutorTests.java | 42 +++++---- .../cluster/coordination/NodeJoinTests.java | 4 +- .../MetadataCreateIndexServiceTests.java | 25 ++--- .../discovery/DiscoveryModuleTests.java | 6 +- .../indices/cluster/ClusterStateChanges.java | 6 +- .../BlobStoreRepositoryHelperTests.java | 2 +- .../BlobStoreRepositoryRemoteIndexTests.java | 64 +++++++------ .../snapshots/SnapshotResiliencyTests.java | 6 +- .../AbstractCoordinatorTestCase.java | 6 +- .../opensearch/test/InternalTestCluster.java | 7 ++ 43 files changed, 366 insertions(+), 485 deletions(-) rename server/src/main/java/org/opensearch/action/admin/cluster/remotestore/{RemoteStoreService.java => RemoteStoreNodeService.java} (88%) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 3c4fe822005e0..07d9b278f6eb9 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -92,10 +92,10 @@ ${path.logs} # cluster.remote_store.enabled: true # # Repository to use for segment upload while enforcing remote store for an index -# cluster.remote_store.segment.repository: my-repo-1 +# node.attr.remote_store.segment.repository : my-repo-1 # # Repository to use for translog upload while enforcing remote store for an index -# cluster.remote_store.translog.repository: my-repo-1 +# node.attr.remote_store.translog.repository : my-repo-1 # # ---------------------------------- Experimental Features ----------------------------------- # diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 6d2d8df106513..83d8e282c3474 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -26,10 +26,15 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { @@ -47,7 +52,6 @@ protected Settings featureFlagSettings() { public void setup() { FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REPOSITORY_NAME, TRANSLOG_REPOSITORY_NAME)); } @Override @@ -66,6 +70,43 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas) { .build(); } + public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { + String segmentRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + String translogRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + TRANSLOG_REPOSITORY_NAME + ); + String segmentRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); + String translogRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + TRANSLOG_REPOSITORY_NAME + ); + + return Settings.builder() + .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) + .put(segmentRepoTypeAttributeKey, "mock") + .put(segmentRepoSettingsAttributeKeyPrefix + "location", repoLocation) + .put(segmentRepoSettingsAttributeKeyPrefix + "random_control_io_exception_rate", ioFailureRate) + .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_verification_file", true) + .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_list_blobs", true) + .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_blobs", skipExceptionBlobList) + .put(segmentRepoSettingsAttributeKeyPrefix + "max_failure_number", maxFailure) + .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME) + .put(translogRepoTypeAttributeKey, "mock") + .put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation) + .build(); + } + protected void deleteRepo() { logger.info("--> Deleting the repository={}", REPOSITORY_NAME); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); @@ -73,27 +114,20 @@ protected void deleteRepo() { assertAcked(clusterAdmin().prepareDeleteRepository(TRANSLOG_REPOSITORY_NAME)); } - protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { + protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) throws Exception { logger.info("--> Creating repository={} at the path={}", REPOSITORY_NAME, repoLocation); + logger.info("--> Creating repository={} at the path={}", TRANSLOG_REPOSITORY_NAME, repoLocation); // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the // repository creation can happen without failure. - createRepository( - REPOSITORY_NAME, - "mock", - Settings.builder() - .put("location", repoLocation) - .put("random_control_io_exception_rate", ioFailureRate) - .put("skip_exception_on_verification_file", true) - .put("skip_exception_on_list_blobs", true) - // Skipping is required for metadata as it is part of recovery - .put("skip_exception_on_blobs", skipExceptionBlobList) - .put("max_failure_number", maxFailure) - ); - logger.info("--> Creating repository={} at the path={}", TRANSLOG_REPOSITORY_NAME, repoLocation); - createRepository(TRANSLOG_REPOSITORY_NAME, "mock", Settings.builder().put("location", repoLocation)); + Settings settings = Settings.builder() + .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomBoolean() ? ReplicationType.SEGMENT : ReplicationType.DOCUMENT) + .put(buildRemoteStoreNodeAttributes(repoLocation, ioFailureRate, skipExceptionBlobList, maxFailure)) + .build(); - String dataNodeName = internalCluster().startDataOnlyNodes(1).get(0); + internalCluster().startClusterManagerOnlyNode(settings); + String dataNodeName = internalCluster().startDataOnlyNode(settings); + // assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME); logger.info("--> Created index={}", INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java index 37dab5faaeb57..e1ab101fddf55 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java @@ -75,8 +75,8 @@ public void testDefaultRemoteStoreNoUserOverrideExceptReplicationTypeSegment() t verifyRemoteStoreIndexSettings( indexSettings, "true", - "my-segment-repo-1", - "my-translog-repo-1", + REPOSITORY_NAME, + REPOSITORY_2_NAME, ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index 9991126bb790c..d427a4db84ba2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -13,12 +13,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.After; import org.junit.Before; import java.util.Locale; @@ -28,53 +25,15 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) -public class CreateRemoteIndexIT extends OpenSearchIntegTestCase { - - @After - public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository("my-segment-repo-1")); - assertAcked(clusterAdmin().prepareDeleteRepository("my-translog-repo-1")); - assertAcked(clusterAdmin().prepareDeleteRepository("my-custom-repo")); - } - - @Override - protected Settings nodeSettings(int nodeOriginal) { - Settings settings = super.nodeSettings(nodeOriginal); - Settings.Builder builder = Settings.builder() - .put(remoteStoreClusterSettings("my-segment-repo-1", "my-translog-repo-1")) - .put(settings); - return builder.build(); - } - - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); - } +public class CreateRemoteIndexIT extends RemoteStoreBaseIntegTestCase { @Before - public void setup() { - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); - assertAcked( - clusterAdmin().preparePutRepository("my-segment-repo-1") - .setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath().toAbsolutePath())) - ); - assertAcked( - clusterAdmin().preparePutRepository("my-translog-repo-1") - .setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath().toAbsolutePath())) - ); - assertAcked( - clusterAdmin().preparePutRepository("my-custom-repo") - .setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath().toAbsolutePath())) - ); + public void setup() throws Exception { + internalCluster().startNodes(2); } public void testDefaultRemoteStoreNoUserOverride() throws Exception { @@ -91,8 +50,8 @@ public void testDefaultRemoteStoreNoUserOverride() throws Exception { verifyRemoteStoreIndexSettings( indexSettings, "true", - "my-segment-repo-1", - "my-translog-repo-1", + REPOSITORY_NAME, + REPOSITORY_2_NAME, ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index a6d8a987886af..21c8dcfb333ce 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -27,7 +27,9 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.disruption.NetworkDisruption; import org.opensearch.test.transport.MockTransportService; +import org.junit.Before; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -43,12 +45,20 @@ public class PrimaryTermValidationIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; + protected Path absolutePath; + protected Path absolutePath2; @Override protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } + @Before + public void setup() { + absolutePath = randomRepoPath().toAbsolutePath(); + absolutePath2 = randomRepoPath().toAbsolutePath(); + } + public void testPrimaryTermValidation() throws Exception { // Follower checker interval is lower compared to leader checker so that the cluster manager can remove the node // with network partition faster. The follower check retry count is also kept 1. @@ -59,12 +69,10 @@ public void testPrimaryTermValidation() throws Exception { .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "1s") .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "1s") .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath, REPOSITORY_2_NAME, absolutePath2)) .build(); internalCluster().startClusterManagerOnlyNode(clusterSettings); internalCluster().startDataOnlyNodes(2, clusterSettings); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); // Create index createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java index e4dcd637ac448..345fe7a6db77a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java @@ -18,7 +18,6 @@ import java.nio.file.Path; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) public class RemoteIndexPrimaryRelocationIT extends IndexPrimaryRelocationIT { @@ -29,15 +28,12 @@ public class RemoteIndexPrimaryRelocationIT extends IndexPrimaryRelocationIT { public void setup() { absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); } protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_NAME, false)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java index a48ab7b828157..e9539bb4fe049 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -15,18 +15,14 @@ import org.opensearch.index.IndexSettings; import org.opensearch.indices.recovery.IndexRecoveryIT; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.After; +import org.junit.Before; -import java.util.Locale; +import java.nio.file.Path; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -35,33 +31,18 @@ public class RemoteIndexRecoveryIT extends IndexRecoveryIT { protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path repositoryPath; + + @Before + public void setup() { + repositoryPath = randomRepoPath().toAbsolutePath(); + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME)) - .put(repositoryNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, randomRepoPath().toAbsolutePath().toString())) - .build(); - } - - private Settings repositoryNodeAttributes(String name, String type, String location) { - String segmentRepoNameAttributeKey = "node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; - String translogRepoNameAttributeKey = "node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; - String typeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - name - ); - String settingsAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - name - ); - return Settings.builder() - .put(segmentRepoNameAttributeKey, name) - .put(translogRepoNameAttributeKey, name) - .put(typeAttributeKey, type) - .put(settingsAttributeKey + "location", location) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, repositoryPath, REPOSITORY_NAME, repositoryPath)) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 70e571604ca53..11039742f5d20 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -29,7 +29,6 @@ import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.InternalTestCluster; import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.nio.file.Path; @@ -47,23 +46,21 @@ public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase { private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; private Path remoteRepoPath; - @Before - public void setup() { - remoteRepoPath = randomRepoPath().toAbsolutePath(); - createRepository(BASE_REMOTE_REPO, "fs", remoteRepoPath); - } - @After public void teardown() { + remoteRepoPath = null; assertAcked(clusterAdmin().prepareDeleteRepository(BASE_REMOTE_REPO)); } @Override protected Settings nodeSettings(int nodeOrdinal) { + if (remoteRepoPath == null) { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + } return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(FeatureFlags.REMOTE_STORE, "true") - .put(remoteStoreClusterSettings(BASE_REMOTE_REPO)) + .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java index 7e273828a86ef..32f3b1066aacd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java @@ -25,15 +25,14 @@ public class RemoteSegmentStatsFromNodesStatsIT extends RemoteStoreBaseIntegTest private static final int CLUSTER_MANAGER_NODE_COUNT = 3; @Before - public void setup() throws Exception { + public void setup() { setupCustomCluster(); } - private void setupCustomCluster() throws Exception { + private void setupCustomCluster() { internalCluster().startClusterManagerOnlyNodes(CLUSTER_MANAGER_NODE_COUNT); internalCluster().startDataOnlyNodes(DATA_NODE_COUNT); ensureStableCluster(DATA_NODE_COUNT + CLUSTER_MANAGER_NODE_COUNT); - assertRepositoryMetadataPresentInClusterState(); } /** diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index bdc2ef2dedefc..53bb95021f0c9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -16,7 +16,6 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -38,16 +37,12 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -117,10 +112,15 @@ protected boolean addMockInternalEngine() { @Override protected Settings nodeSettings(int nodeOrdinal) { + Settings nodeAttributes = Settings.EMPTY; + if (segmentRepoPath == null || translogRepoPath == null) { + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + } return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) - .put(remoteStoreNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, REPOSITORY_2_NAME, FsRepository.TYPE)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)) + .put(nodeAttributes) .build(); } @@ -163,81 +163,62 @@ protected BulkResponse indexBulk(String indexName, int numDocs) { return client().bulk(bulkRequest).actionGet(); } - public static Settings remoteStoreClusterSettings(String segmentRepoName) { - return remoteStoreClusterSettings(segmentRepoName, segmentRepoName); + public static Settings remoteStoreClusterSettings(String name, Path path) { + return remoteStoreClusterSettings(name, path, name, path); } public static Settings remoteStoreClusterSettings( String segmentRepoName, + Path segmentRepoPath, String translogRepoName, - boolean randomizeSameRepoForRSSAndRTS + Path translogRepoPath ) { - return remoteStoreClusterSettings( - segmentRepoName, - randomizeSameRepoForRSSAndRTS ? (randomBoolean() ? translogRepoName : segmentRepoName) : translogRepoName - ); - } - - public static Settings remoteStoreClusterSettings(String segmentRepoName, String translogRepoName) { - Settings.Builder settingsBuilder = Settings.builder() - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), segmentRepoName) - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), translogRepoName); + Settings.Builder settingsBuilder = Settings.builder(); if (randomBoolean()) { settingsBuilder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT); } + settingsBuilder.put(buildRemoteStoreNodeAttributes(segmentRepoName, segmentRepoPath, translogRepoName, translogRepoPath)); + return settingsBuilder.build(); } - public Settings remoteStoreNodeAttributes( + public static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, - String segmentRepoType, + Path segmentRepoPath, String translogRepoName, - String translogRepoType + Path translogRepoPath ) { - if (nodeAttributesSettings != null) { - return nodeAttributesSettings; - } - segmentRepoPath = randomRepoPath().toAbsolutePath(); - translogRepoPath = randomRepoPath().toAbsolutePath(); - String segmentRepoKey = String.format( + String segmentRepoTypeAttributeKey = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, segmentRepoName ); - String translogRepoKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - translogRepoName - ); - String segmentRepoSettingsPrefix = String.format( + String segmentRepoSettingsAttributeKeyPrefix = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, segmentRepoName ); - String translogRepoSettingsPrefix = String.format( + String translogRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + translogRepoName + ); + String translogRepoSettingsAttributeKeyPrefix = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName ); - if (segmentRepoName.equals(translogRepoName)) { - segmentRepoPath = translogRepoPath; - segmentRepoKey = translogRepoKey; - segmentRepoSettingsPrefix = translogRepoSettingsPrefix; - } - - nodeAttributesSettings = Settings.builder() + return Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(segmentRepoKey, segmentRepoType) - .put(segmentRepoSettingsPrefix + "location", segmentRepoPath.toString()) + .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) + .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) - .put(translogRepoKey, translogRepoType) - .put(translogRepoSettingsPrefix + "location", translogRepoPath.toString()) + .put(translogRepoTypeAttributeKey, FsRepository.TYPE) + .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .build(); - return nodeAttributesSettings; } private Settings defaultIndexSettings() { @@ -295,17 +276,4 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { return filesExisting.get(); } - - public void assertRepositoryMetadataPresentInClusterState() throws Exception { - assertBusy(() -> { - RepositoriesMetadata repositoriesMetadata = client().admin() - .cluster() - .prepareState() - .get() - .getState() - .metadata() - .custom(RepositoriesMetadata.TYPE); - assertTrue(repositoriesMetadata != null && !repositoriesMetadata.repositories().isEmpty()); - }, 30, TimeUnit.SECONDS); - } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index 61497ed896cf5..0bcde4b44c734 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -17,6 +17,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -89,10 +90,8 @@ private void verifyRestoredData(Map indexStats, long deletedDocs) } private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) - throws Exception { + throws IOException { internalCluster().startNodes(3); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -124,19 +123,19 @@ private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlus // values for each of the flags, number of integ tests become 16 in comparison to current 2. // We have run all the 16 tests on local and they run fine. @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9294") - public void testRestoreForceMergeSingleIteration() throws Exception { + public void testRestoreForceMergeSingleIteration() throws IOException { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(1, invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); } - public void testRestoreForceMergeMultipleIterations() throws Exception { + public void testRestoreForceMergeMultipleIterations() throws IOException { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(randomIntBetween(2, 5), invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); } - public void testRestoreForceMergeMultipleIterationsDeleteAll() throws Exception { + public void testRestoreForceMergeMultipleIterationsDeleteAll() throws IOException { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(randomIntBetween(2, 3), invokeFLush, flushAfterMerge, -1); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index e6bcf094e56c1..345b8369befef 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -63,8 +63,6 @@ public Settings indexSettings() { private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startNodes(3); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -124,10 +122,8 @@ public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogRefresh() throws Exc testPeerRecovery(randomIntBetween(2, 5), false); } - public void verifyRemoteStoreCleanup() throws Exception { + private void verifyRemoteStoreCleanup() throws Exception { internalCluster().startNodes(3); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); indexData(5, randomBoolean(), INDEX_NAME); @@ -154,8 +150,6 @@ public void testRemoteTranslogCleanup() throws Exception { public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startNode(); - ensureStableCluster(1); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); @@ -183,8 +177,6 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startNode(); - ensureStableCluster(1); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, false, INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 1ca6d2c23cf0d..47d3c29756cd0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -15,7 +15,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -55,7 +54,6 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na } private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { - assertRepositoryMetadataPresentInClusterState(); RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) .state() .metadata() @@ -75,27 +73,17 @@ private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { public void testSingleNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNode(); - ensureStableCluster(1); assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNodes(3); - ensureStableCluster(3); assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() throws Exception { internalCluster().startClusterManagerOnlyNodes(3); internalCluster().startNodes(3); - ensureStableCluster(6); - assertRemoteStoreRepositoryOnAllNodes(); - } - - public void testMultiNodeClusterRepositoryRegistrationDifferent() throws Exception { - internalCluster().startNodes(3); - internalCluster().startNode(remoteStoreNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, REPOSITORY_NAME, FsRepository.TYPE)); - ensureStableCluster(3); assertRemoteStoreRepositoryOnAllNodes(); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 4b6c98ba60afe..aeb4756c21a8b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -87,11 +87,9 @@ private void verifyRestoredData(Map indexStats, String indexName) ); } - private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) - throws Exception { + private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); internalCluster().startDataOnlyNodes(numDataOnlyNodes); - assertRepositoryMetadataPresentInClusterState(); for (String index : indices.split(",")) { createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); ensureYellowAndNoInitializingShards(index); @@ -263,7 +261,7 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo } } - public void testRestoreFlowAllShardsNoRedIndex() throws Exception { + public void testRestoreFlowAllShardsNoRedIndex() throws InterruptedException { int shardCount = randomIntBetween(1, 5); prepareCluster(1, 3, INDEX_NAME, 0, shardCount); indexData(randomIntBetween(2, 5), true, INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 6999b2d9a4858..7383256215630 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -48,10 +48,9 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { public void setup() throws Exception { internalCluster().startNodes(3); ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); } - public void testStatsResponseFromAllNodes() throws Exception { + public void testStatsResponseFromAllNodes() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. @@ -119,7 +118,7 @@ public void testStatsResponseFromAllNodes() throws Exception { } } - public void testStatsResponseAllShards() throws Exception { + public void testStatsResponseAllShards() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. @@ -176,7 +175,7 @@ public void testStatsResponseAllShards() throws Exception { } - public void testStatsResponseFromLocalNode() throws Exception { + public void testStatsResponseFromLocalNode() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index edea93464e650..3e41c4925ea46 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -45,8 +45,6 @@ public Settings indexSettings() { public void testPromoteReplicaToPrimary() throws Exception { internalCluster().startNode(); internalCluster().startNode(); - ensureStableCluster(2); - assertRepositoryMetadataPresentInClusterState(); final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); shard_count = scaledRandomIntBetween(1, 5); createIndex(indexName); @@ -122,8 +120,6 @@ public void testPromoteReplicaToPrimary() throws Exception { public void testFailoverWhileIndexing() throws Exception { internalCluster().startNode(); internalCluster().startNode(); - ensureStableCluster(2); - assertRepositoryMetadataPresentInClusterState(); final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); shard_count = scaledRandomIntBetween(1, 5); createIndex(indexName); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index 22250c3b793cf..1b817408596ab 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -30,10 +30,17 @@ public class SegmentReplicationUsingRemoteStoreIT extends SegmentReplicationIT { private static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path absolutePath; @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + if (absolutePath == null) { + absolutePath = randomRepoPath().toAbsolutePath(); + } + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) + .build(); } protected boolean segmentReplicationWithRemoteEnabled() { @@ -52,10 +59,6 @@ protected Settings featureFlagSettings() { @Before public void setup() { internalCluster().startClusterManagerOnlyNode(); - Path absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); } @After diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java index 99927797d5fbc..fa0944e5bfee0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java @@ -29,6 +29,7 @@ public class SegmentReplicationWithRemoteStorePressureIT extends SegmentReplicationPressureIT { private static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path absolutePath; @Override protected boolean segmentReplicationWithRemoteEnabled() { @@ -46,16 +47,16 @@ protected Settings featureFlagSettings() { @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) + .build(); } @Before public void setup() { + absolutePath = randomRepoPath().toAbsolutePath(); internalCluster().startClusterManagerOnlyNode(); - Path absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); } @After diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java index da8ba77436f7f..d7d80f90089d9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -34,16 +34,6 @@ protected Collection> nodePlugins() { return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); } - @Override - public Settings remoteStoreNodeAttributes( - String segmentRepoName, - String segmentRepoType, - String translogRepoName, - String translogRepoType - ) { - return super.remoteStoreNodeAttributes(segmentRepoName, MockFsRepositoryPlugin.TYPE, translogRepoName, MockFsRepositoryPlugin.TYPE); - } - public void testRateLimitedRemoteUploads() throws Exception { internalCluster().startNode(); Client client = client(); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 072e03e8a2f79..55ce2805fcb94 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -161,8 +161,9 @@ public void testCloneShallowSnapshotIndex() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); @@ -172,9 +173,6 @@ public void testCloneShallowSnapshotIndex() throws Exception { final Path shallowSnapshotRepoPath = randomRepoPath(); createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -208,10 +206,14 @@ public void testShallowCloneNameAvailability() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; + final Path remoteStorePath = randomRepoPath().toAbsolutePath(); internalCluster().startClusterManagerOnlyNode( - Settings.builder().put(LARGE_SNAPSHOT_POOL_SETTINGS).put(remoteStoreClusterSettings(remoteStoreRepoName)).build() + Settings.builder() + .put(LARGE_SNAPSHOT_POOL_SETTINGS) + .put(remoteStoreClusterSettings(remoteStoreRepoName, remoteStorePath)) + .build() ); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStorePath)); final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; final Path shallowSnapshotRepoPath = randomRepoPath(); @@ -245,16 +247,14 @@ public void testCloneAfterRepoShallowSettingEnabled() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "fs", snapshotRepoPath); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -282,16 +282,14 @@ public void testCloneAfterRepoShallowSettingDisabled() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 3a9eacd6ac183..448b860683668 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -42,16 +42,14 @@ public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase { public void testDeleteSnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "fs", snapshotRepoPath); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -72,15 +70,13 @@ public void testDeleteSnapshot() throws Exception { public void testDeleteShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -104,14 +100,12 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); @@ -150,8 +144,9 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - final String dataNode = internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + final String dataNode = internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); ensureStableCluster(2); final String clusterManagerNode = internalCluster().getClusterManagerName(); @@ -161,9 +156,6 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { final String testIndex = "index-test"; createIndexWithContent(testIndex); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); @@ -238,8 +230,9 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); @@ -247,9 +240,6 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String testIndex = "index-test"; createIndexWithContent(testIndex); @@ -300,8 +290,9 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); @@ -309,9 +300,6 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception { final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String testIndex = "index-test"; createIndexWithContent(testIndex); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java index fb91b1d7a006c..231e8ad3788be 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java @@ -40,7 +40,9 @@ import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; +import org.junit.Before; import java.nio.file.Path; @@ -49,8 +51,17 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) public class RemoteIndexSnapshotStatusApiIT extends AbstractSnapshotIntegTestCase { + protected Path absolutePath; + final String remoteStoreRepoName = "remote-store-repo-name"; + + @Before + public void setup() { + absolutePath = randomRepoPath().toAbsolutePath(); + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -58,7 +69,7 @@ protected Settings nodeSettings(int nodeOrdinal) { .put(ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.getKey(), 0) // We have tests that check by-timestamp order .put(FeatureFlags.REMOTE_STORE, "true") .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") - .put(remoteStoreClusterSettings("remote-store-repo-name")) + .put(remoteStoreClusterSettings(remoteStoreRepoName, absolutePath)) .build(); } @@ -70,10 +81,6 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception { final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); @@ -109,10 +116,6 @@ public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); @@ -157,10 +160,6 @@ public void testStatusAPICallInProgressShallowSnapshot() throws Exception { final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy().put("block_on_data", true)); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "mock", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java similarity index 88% rename from server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java rename to server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java index e2cee2f2f6449..6b6733c7f1ab7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java @@ -30,9 +30,9 @@ /** * Contains all the method needed for a remote store backed node lifecycle. */ -public class RemoteStoreService { +public class RemoteStoreNodeService { - private static final Logger logger = LogManager.getLogger(RemoteStoreService.class); + private static final Logger logger = LogManager.getLogger(RemoteStoreNodeService.class); private final Supplier repositoriesService; private final ThreadPool threadPool; public static final Setting REMOTE_STORE_COMPATIBILITY_MODE_SETTING = new Setting<>( @@ -74,39 +74,37 @@ public static CompatibilityMode parseString(String compatibilityMode) { } } - public RemoteStoreService(Supplier repositoriesService, ThreadPool threadPool) { + public RemoteStoreNodeService(Supplier repositoriesService, ThreadPool threadPool) { this.repositoriesService = repositoriesService; this.threadPool = threadPool; } /** - * Performs repository verification during node startup post its creation by invoking verify method against - * repository mentioned. This verification will happen on a local node to validate if the node is able to connect - * to the repository. + * Creates a repository during a node startup and performs verification by invoking verify method against + * mentioned repository. This verification will happen on a local node to validate if the node is able to connect + * to the repository with appropriate permissions. */ - public void verifyRepositoriesLocally(List repositories, DiscoveryNode localNode) { - for (Repository repository : repositories) { - String repositoryName = repository.getMetadata().name(); - String verificationToken = repository.startVerification(); - repository.verify(verificationToken, localNode); - repository.endVerification(verificationToken); - logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); - } - } - - /** - * Creates a repository during a node startup. - */ - public List createRepositories(RemoteStoreNode node) { + public List createAndVerifyRepositories(DiscoveryNode localNode) { + RemoteStoreNode node = new RemoteStoreNode(localNode); List repositories = new ArrayList<>(); for (RepositoryMetadata repositoryMetadata : node.getRepositoriesMetadata().repositories()) { - RepositoriesService.validate(repositoryMetadata.name()); + String repositoryName = repositoryMetadata.name(); + + // Create Repository + RepositoriesService.validate(repositoryName); Repository repository = repositoriesService.get().createRepository(repositoryMetadata); logger.info( "remote backed storage repository with name {} and type {} created.", repository.getMetadata().name(), repository.getMetadata().type() ); + + // Verify Repository + String verificationToken = repository.startVerification(); + repository.verify(verificationToken, localNode); + repository.endVerification(verificationToken); + logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); + repositories.add(repository); } return repositories; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 01473bc69cae0..321b8a65e4d23 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -183,7 +183,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private Optional currentPublication = Optional.empty(); private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; - private final RemoteStoreService remoteStoreService; + private final RemoteStoreNodeService remoteStoreService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -206,7 +206,7 @@ public Coordinator( ElectionStrategy electionStrategy, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreService remoteStoreService + RemoteStoreNodeService remoteStoreService ) { this.settings = settings; this.transportService = transportService; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 50a824f5ae300..14f30c272a3a4 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -37,7 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateTaskListener; @@ -136,7 +136,7 @@ public class JoinHelper { AllocationService allocationService, ClusterManagerService clusterManagerService, TransportService transportService, - RemoteStoreService remoteStoreService, + RemoteStoreNodeService remoteStoreService, LongSupplier currentTermSupplier, Supplier currentStateSupplier, BiConsumer joinHandler, diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 53b25970a1876..d25f327fdb37d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; import org.opensearch.cluster.NotClusterManagerException; @@ -61,9 +61,9 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode.STRICT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.CompatibilityMode; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -79,7 +79,7 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor execute(ClusterState currentState, List jo final DiscoveryNode node = joinTask.node(); if (joinTask.isBecomeClusterManagerTask() || joinTask.isFinishElectionTask()) { // noop - } else if (currentNodes.nodeExists(node)) { - if (currentNodes.nodeExistsWithSameRoles(node)) { - logger.debug("received a join request for an existing node [{}]", node); - } + } else if (currentNodes.nodeExistsWithSameRoles(node)) { + logger.debug("received a join request for an existing node [{}]", node); + // TODO: Fix this by moving it out of this if condition, Had to add this code back here as this was + // leading to failure of JoinTaskExecutorTests::testUpdatesNodeWithNewRoles test. if (node.isRemoteStoreNode()) { /** cluster state is updated here as elect leader task can have same node present in join task as * well as current node. We want the repositories to be added in cluster state during first node diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index fd7fe29442eb2..9bef81d9019cb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -39,6 +39,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.opensearch.action.admin.indices.shrink.ResizeType; @@ -94,6 +95,7 @@ import org.opensearch.indices.ShardLimitValidator; import org.opensearch.indices.SystemIndices; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.Node; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -132,9 +134,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; /** @@ -927,7 +926,8 @@ static Settings aggregateIndexSettings( } /** - * Updates index settings to set replication strategy by default based on cluster level settings + * Updates index settings to set replication strategy by default based on cluster level settings or remote store + * node attributes * @param settingsBuilder index settings builder to be updated with relevant settings * @param requestSettings settings passed in during index create request * @param clusterSettings cluster level settings @@ -937,23 +937,31 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, settingsBuilder.put(SETTING_REPLICATION_TYPE, INDEX_REPLICATION_TYPE_SETTING.get(requestSettings)); } else if (CLUSTER_REPLICATION_TYPE_SETTING.exists(clusterSettings)) { settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.get(clusterSettings)); - } else if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings)) { - settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); - } else { - settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings)); - } + } else if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) + .isEmpty() == false) { + settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); + } else { + settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings)); + } } /** - * Updates index settings to enable remote store by default based on cluster level settings + * Updates index settings to enable remote store by default based on node attributes * @param settingsBuilder index settings builder to be updated with relevant settings * @param clusterSettings cluster level settings */ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings clusterSettings) { - if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings) == true) { + if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) + .isEmpty() == false) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(clusterSettings)) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings)); + .put( + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + clusterSettings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) + ) + .put( + SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + clusterSettings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY) + ); } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 30f5be2730ac9..1df6984c5659e 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -33,8 +33,7 @@ package org.opensearch.cluster.node; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; @@ -46,13 +45,11 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; -import org.opensearch.repositories.Repository; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -287,11 +284,11 @@ public static DiscoveryNode createLocal(Settings settings, TransportAddress publ } /** Creates a DiscoveryNode representing the local node and verifies the repository. */ - public static DiscoveryNode createLocal( + public static DiscoveryNode createRemoteNodeLocal( Settings settings, TransportAddress publishAddress, String nodeId, - RemoteStoreService remoteStoreService + RemoteStoreNodeService remoteStoreService ) { Map attributes = Node.NODE_ATTRIBUTES.getAsMap(settings); Set roles = getRolesFromSettings(settings); @@ -303,9 +300,7 @@ public static DiscoveryNode createLocal( roles, Version.CURRENT ); - RemoteStoreNode remoteStoreNode = new RemoteStoreNode(discoveryNode); - List repositories = remoteStoreService.createRepositories(remoteStoreNode); - remoteStoreService.verifyRepositoriesLocally(repositories, discoveryNode); + remoteStoreService.createAndVerifyRepositories(discoveryNode); return discoveryNode; } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 1077a103fdc4b..7c5efb8f19df0 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.LogManager; import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.indices.close.TransportCloseIndexAction; import org.opensearch.action.search.CreatePitController; import org.opensearch.action.search.TransportSearchAction; @@ -671,7 +671,7 @@ public void apply(Settings value, Settings current, Settings previous) { // Remote cluster state settings RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING, - RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING + RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING ) ) ); @@ -684,13 +684,6 @@ public void apply(Settings value, Settings current, Settings previous) { * setting should be moved to {@link #BUILT_IN_CLUSTER_SETTINGS}. */ public static final Map, List> FEATURE_FLAGGED_CLUSTER_SETTINGS = Map.of( - List.of(FeatureFlags.REMOTE_STORE), - List.of( - IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING, - IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, - IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING, - IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING - ), List.of(FeatureFlags.CONCURRENT_SEGMENT_SEARCH), List.of( SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING, diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index dd97ba1a4db3a..cead506349169 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.coordination.ElectionStrategy; @@ -133,7 +133,7 @@ public DiscoveryModule( RerouteService rerouteService, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreService remoteStoreService + RemoteStoreNodeService remoteStoreService ) { final Collection> joinValidators = new ArrayList<>(); final Map> hostProviders = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 8defaef1c844b..3507c485e5427 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -243,47 +243,6 @@ public class IndicesService extends AbstractLifecycleComponent Property.Final ); - /** - * Used to specify if all indexes are to create with remote store enabled by default - */ - public static final Setting CLUSTER_REMOTE_STORE_ENABLED_SETTING = Setting.boolSetting( - "cluster.remote_store.enabled", - false, - Property.NodeScope, - Property.Final - ); - - /** - * Used to specify default repo to use for segment upload for remote store backed indices - */ - public static final Setting CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString( - "cluster.remote_store.segment.repository", - "", - Property.NodeScope, - Property.Final - ); - - /** - * Used to specify default repo to use for translog upload for remote store backed indices - */ - public static final Setting CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING = Setting.simpleString( - "cluster.remote_store.translog.repository", - "", - Property.NodeScope, - Property.Final - ); - - /** - * Used to specify the default translog buffer interval for remote store backed indexes. - */ - public static final Setting CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING = Setting.timeSetting( - "cluster.remote_store.translog.buffer_interval", - IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, - IndexSettings.MINIMUM_REMOTE_TRANSLOG_BUFFER_INTERVAL, - Property.NodeScope, - Property.Dynamic - ); - /** * This setting is used to set the refresh interval when the {@code index.refresh_interval} index setting is not * provided during index creation or when the existing {@code index.refresh_interval} index setting is set as null. diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index a088cb743bdb5..0522a3604db34 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -43,8 +43,7 @@ import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; import org.opensearch.action.search.SearchPhaseController; @@ -268,6 +267,7 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.common.util.FeatureFlags.TELEMETRY; import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; @@ -531,7 +531,7 @@ protected Node( final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, executorBuilders.toArray(new ExecutorBuilder[0])); final SetOnce repositoriesServiceReference = new SetOnce<>(); - final RemoteStoreService remoteStoreService = new RemoteStoreService(repositoriesServiceReference::get, threadPool); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService(repositoriesServiceReference::get, threadPool); localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId(), remoteStoreService); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); @@ -1744,9 +1744,9 @@ private static class LocalNodeFactory implements Function localNode = new SetOnce<>(); private final String persistentNodeId; private final Settings settings; - private final RemoteStoreService remoteStoreService; + private final RemoteStoreNodeService remoteStoreService; - private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStoreService remoteStoreService) { + private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStoreNodeService remoteStoreService) { this.persistentNodeId = persistentNodeId; this.settings = settings; this.remoteStoreService = remoteStoreService; @@ -1757,9 +1757,14 @@ public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { if (Node.NODE_ATTRIBUTES.getAsMap(settings) .keySet() .stream() - .anyMatch(key -> key.startsWith(RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX))) { + .anyMatch(key -> key.startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX))) { localNode.set( - DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId, remoteStoreService) + DiscoveryNode.createRemoteNodeLocal( + settings, + boundTransportAddress.publishAddress(), + persistentNodeId, + remoteStoreService + ) ); } else { localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId)); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java index d3fbb0cfce09d..5d6c632b3a499 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Level; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -509,7 +509,7 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin return new TestClusterSetup(deterministicTaskQueue, localNode, transportService, localClusterState, joinHelper, capturingTransport); } - private RemoteStoreService buildRemoteStoreService(TransportService transportService, ThreadPool threadPool) { + private RemoteStoreNodeService buildRemoteStoreService(TransportService transportService, ThreadPool threadPool) { RepositoriesService repositoriesService = new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), @@ -518,7 +518,7 @@ private RemoteStoreService buildRemoteStoreService(TransportService transportSer Collections.emptyMap(), threadPool ); - return new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + return new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); } private static class TestClusterSetup { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index a886bf172c508..fc66f9bd45dda 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -33,7 +33,7 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; @@ -54,6 +54,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -187,7 +188,7 @@ public void testUpdatesNodeWithNewRoles() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = mock(RemoteStoreService.class); + final RemoteStoreNodeService remoteStoreService = mock(RemoteStoreNodeService.class); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -291,7 +292,7 @@ public void testJoinFailedForDecommissionedNode() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = mock(RemoteStoreService.class); + final RemoteStoreNodeService remoteStoreService = mock(RemoteStoreNodeService.class); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -427,7 +428,7 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentAttributesJoin if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY) { remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue() + "-new"); - validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue()); } } @@ -449,10 +450,10 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentNameAttributes for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { if (REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO + "new", TRANSLOG_REPO); - validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); } else if (REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO + "new"); - validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); } } } @@ -511,7 +512,10 @@ public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -553,7 +557,10 @@ public void testUpdatesClusterStateWithMultiNodeCluster() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -613,7 +620,10 @@ public void testUpdatesClusterStateWithSingleNodeClusterAndSameRepository() thro final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -655,7 +665,10 @@ public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throw final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -782,12 +795,7 @@ private Map remoteStoreNodeAttributes(String segmentRepoName, St }; } - private void validateAttributes( - Map remoteStoreNodeAttributes, - Map.Entry existingNodeAttribute, - ClusterState currentState, - DiscoveryNode existingNode - ) { + private void validateAttributes(Map remoteStoreNodeAttributes, ClusterState currentState, DiscoveryNode existingNode) { DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); Exception e = assertThrows( IllegalStateException.class, @@ -820,6 +828,8 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na Settings.Builder settings = Settings.builder(); settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); + return new RepositoryMetadata(name, type, settings.build()); } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java index fa4bbcc934906..bd1e6d9b9ec71 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.OpenSearchAllocationTestCase; @@ -268,7 +268,7 @@ protected void onSendRequest( ElectionStrategy.DEFAULT_INSTANCE, nodeHealthService, persistedStateRegistry, - Mockito.mock(RemoteStoreService.class) + Mockito.mock(RemoteStoreNodeService.class) ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index ad5fbb18138b3..820836f41249d 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -114,6 +114,8 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK; @@ -136,11 +138,9 @@ import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; +import static org.opensearch.node.Node.NODE_ATTRIBUTES; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -158,6 +158,10 @@ public class MetadataCreateIndexServiceTests extends OpenSearchTestCase { private CreateIndexClusterStateUpdateRequest request; private QueryShardContext queryShardContext; private ClusterSettings clusterSettings; + private static final String segmentRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() + + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; + private static final String translogRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() + + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; @Before public void setup() throws Exception { @@ -1214,9 +1218,8 @@ public void testvalidateIndexSettings() { public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1") + .put(translogRepositoryNameAttributeKey, "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1247,9 +1250,8 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin public void testRemoteStoreImplicitOverrideReplicationTypeToSegmentForRemoteStore() { Settings settings = Settings.builder() - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1") + .put(translogRepositoryNameAttributeKey, "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1280,9 +1282,8 @@ public void testRemoteStoreImplicitOverrideReplicationTypeToSegmentForRemoteStor public void testRemoteStoreNoUserOverrideIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1") + .put(translogRepositoryNameAttributeKey, "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index 2e230ca5514cc..8cd742d5104ea 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -32,7 +32,7 @@ package org.opensearch.discovery; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.coordination.PersistedStateRegistry; @@ -78,7 +78,7 @@ public class DiscoveryModuleTests extends OpenSearchTestCase { private ClusterSettings clusterSettings; private GatewayMetaState gatewayMetaState; - private RemoteStoreService remoteStoreService; + private RemoteStoreNodeService remoteStoreService; public interface DummyHostsProviderPlugin extends DiscoveryPlugin { Map> impl(); @@ -102,7 +102,7 @@ public void setupDummyServices() { clusterApplier = mock(ClusterApplier.class); clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); gatewayMetaState = mock(GatewayMetaState.class); - remoteStoreService = mock(RemoteStoreService.class); + remoteStoreService = mock(RemoteStoreNodeService.class); } @After diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 06b604152bf11..31bf54e9bcc0c 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.ExceptionsHelper; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest; import org.opensearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.opensearch.action.admin.indices.close.CloseIndexRequest; @@ -154,7 +154,7 @@ public class ClusterStateChanges { private final TransportClusterRerouteAction transportClusterRerouteAction; private final TransportCreateIndexAction transportCreateIndexAction; private final RepositoriesService repositoriesService; - private final RemoteStoreService remoteStoreService; + private final RemoteStoreNodeService remoteStoreService; private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor; private final JoinTaskExecutor joinTaskExecutor; @@ -376,7 +376,7 @@ public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version m threadPool ); - remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger); joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, (s, p, r) -> {}, remoteStoreService); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java index 0dbc0372458b5..a24fd04d3d4f6 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java @@ -111,7 +111,7 @@ protected void updateRepository(Client client, String repoName, Settings repoSet createRepository(client, repoName, repoSettings); } - protected Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { + protected Settings getRemoteStoreBackedIndexSettings() { return Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") .put("index.refresh_interval", "300s") diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index d69fb1fd7b349..d59a33b4cbbdd 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -39,24 +39,29 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; @@ -71,12 +76,36 @@ protected Settings featureFlagSettings() { @Override protected Settings nodeSettings() { + Path tempDir = createTempDir(); return Settings.builder() .put(super.nodeSettings()) .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "test-rs-repo") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "test-rs-repo") + .put(buildRemoteStoreNodeAttributes("test-rs-repo", tempDir.resolve("repo"))) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo")) + .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), tempDir.getParent()) + .build(); + } + + public static Settings buildRemoteStoreNodeAttributes(String repoName, Path repoPath) { + String repoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + repoName + ); + String repoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + repoName + ); + + return Settings.builder() + .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put(repoTypeAttributeKey, FsRepository.TYPE) + .put(repoSettingsAttributeKeyPrefix + "location", repoPath) + .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put(repoTypeAttributeKey, FsRepository.TYPE) + .put(repoSettingsAttributeKeyPrefix + "location", repoPath) .build(); } @@ -95,13 +124,6 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { .build(); createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - logger.info("--> creating remote store repository"); - Settings remoteStoreRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); - logger.info("--> creating an index and indexing documents"); final String indexName = "test-idx"; createIndex(indexName); @@ -110,7 +132,7 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { logger.info("--> creating a remote store enabled index and indexing documents"); final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + Settings indexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreIndexName, indexSettings); indexDocuments(client, remoteStoreIndexName); @@ -195,16 +217,9 @@ public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException { .build(); createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - logger.info("--> creating remote store repository"); - Settings remoteStoreRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); - logger.info("--> creating a remote store enabled index and indexing documents"); final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + Settings indexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreIndexName, indexSettings); indexDocuments(client, remoteStoreIndexName); @@ -266,9 +281,6 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { assertFalse(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); - logger.info("--> creating remote store repository"); - createRepository(client, remoteStoreRepositoryName); - logger.info("--> creating an index and indexing documents"); final String indexName = "test-idx"; createIndex(indexName); @@ -277,7 +289,7 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { logger.info("--> creating a remote store enabled index and indexing documents"); final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + Settings indexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreIndexName, indexSettings); indexDocuments(client, remoteStoreIndexName); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 75d2602609e4b..59ebbf7204b8a 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.RequestValidators; import org.opensearch.action.StepListener; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -1893,7 +1893,7 @@ private final class TestClusterNode { private final ClusterInfoService clusterInfoService; private Coordinator coordinator; - private RemoteStoreService remoteStoreService; + private RemoteStoreNodeService remoteStoreService; private Map actions = new HashMap<>(); @@ -1999,7 +1999,7 @@ public void onFailure(final Exception e) { emptyMap(), threadPool ); - remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService( settings, diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 347dd3cdfab4f..478dd8d3f43de 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -39,7 +39,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskListener; @@ -1039,7 +1039,7 @@ class ClusterNode { private DisruptableMockTransport mockTransport; private NodeHealthService nodeHealthService; private RepositoriesService repositoriesService; - private RemoteStoreService remoteStoreService; + private RemoteStoreNodeService remoteStoreService; List> extraJoinValidators = new ArrayList<>(); ClusterNode(int nodeIndex, boolean clusterManagerEligible, Settings nodeSettings, NodeHealthService nodeHealthService) { @@ -1142,7 +1142,7 @@ protected Optional getDisruptableMockTransport(Transpo Collections.emptyMap(), threadPool ); - remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); final Collection> onJoinValidators = Collections.singletonList( (dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)) ); diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 11e847e29a097..95832dc9544ce 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -55,6 +55,7 @@ import org.opensearch.cluster.coordination.ClusterBootstrapService; import org.opensearch.cluster.coordination.NoClusterManagerBlockService; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; @@ -1318,6 +1319,12 @@ public synchronized void validateClusterFormed() { assertTrue("Expected node to exist: " + expectedNode + debugString, discoveryNodes.nodeExists(expectedNode)); } }); + states.forEach(cs -> { + if (cs.nodes().getNodes().values().stream().findFirst().get().isRemoteStoreNode()) { + RepositoriesMetadata repositoriesMetadata = cs.metadata().custom(RepositoriesMetadata.TYPE); + assertTrue(repositoriesMetadata != null && !repositoriesMetadata.repositories().isEmpty()); + } + }); }, 30, TimeUnit.SECONDS); } catch (AssertionError ae) { throw new IllegalStateException("cluster failed to form", ae); From 8d9df13b323f6dbb6cec14aa0fc7ee0d245cee62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Fri, 1 Sep 2023 02:59:54 +0530 Subject: [PATCH 16/34] Removing unnecessary diffs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteStoreBaseIntegTestCase.java | 6 +----- .../remotestore/RemoteStoreForceMergeIT.java | 2 +- .../org/opensearch/remotestore/RemoteStoreStatsIT.java | 10 ++++------ .../cluster/coordination/JoinTaskExecutor.java | 6 +++--- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 53bb95021f0c9..d2793eaeb3e5f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -58,7 +58,6 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected Path segmentRepoPath; protected Path translogRepoPath; - protected Settings nodeAttributesSettings; private final List documentKeys = List.of( randomAlphaOfLength(5), randomAlphaOfLength(5), @@ -112,7 +111,6 @@ protected boolean addMockInternalEngine() { @Override protected Settings nodeSettings(int nodeOrdinal) { - Settings nodeAttributes = Settings.EMPTY; if (segmentRepoPath == null || translogRepoPath == null) { segmentRepoPath = randomRepoPath().toAbsolutePath(); translogRepoPath = randomRepoPath().toAbsolutePath(); @@ -120,7 +118,6 @@ protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)) - .put(nodeAttributes) .build(); } @@ -253,8 +250,7 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel } @After - public void teardown() throws Exception { - nodeAttributesSettings = null; + public void teardown() { assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index 0bcde4b44c734..950ef5c3b722c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -91,7 +91,7 @@ private void verifyRestoredData(Map indexStats, long deletedDocs) private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) throws IOException { - internalCluster().startNodes(3); + // internalCluster().startNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 7383256215630..8ae25c6758195 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -45,9 +45,8 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; @Before - public void setup() throws Exception { + public void setup() { internalCluster().startNodes(3); - ensureStableCluster(3); } public void testStatsResponseFromAllNodes() { @@ -416,7 +415,7 @@ public void testDownloadStatsCorrectnessSinglePrimaryMultipleReplicaShards() thr } } - public void testStatsOnShardRelocation() throws Exception { + public void testStatsOnShardRelocation() { // Scenario: // - Create index with single primary and single replica shard // - Index documents @@ -497,7 +496,7 @@ public void testStatsOnShardUnassigned() throws IOException { indexSingleDoc(INDEX_NAME); } - public void testStatsOnRemoteStoreRestore() throws Exception { + public void testStatsOnRemoteStoreRestore() throws IOException { // Creating an index with primary shard count == total nodes in cluster and 0 replicas int dataNodeCount = client().admin().cluster().prepareHealth().get().getNumberOfDataNodes(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0, dataNodeCount)); @@ -512,8 +511,7 @@ public void testStatsOnRemoteStoreRestore() throws Exception { ensureRed(INDEX_NAME); // Start another data node to fulfil the previously launched capacity - String nodeId = internalCluster().startDataOnlyNode(); - ensureStableCluster(3); + internalCluster().startDataOnlyNode(); // Restore index from remote store assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index d25f327fdb37d..fbec77548aaca 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -187,9 +187,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo final DiscoveryNode node = joinTask.node(); if (joinTask.isBecomeClusterManagerTask() || joinTask.isFinishElectionTask()) { // noop - } else if (currentNodes.nodeExistsWithSameRoles(node)) { - logger.debug("received a join request for an existing node [{}]", node); - + } else if (currentNodes.nodeExists(node)) { // TODO: Fix this by moving it out of this if condition, Had to add this code back here as this was // leading to failure of JoinTaskExecutorTests::testUpdatesNodeWithNewRoles test. if (node.isRemoteStoreNode()) { @@ -201,6 +199,8 @@ public ClusterTasksResult execute(ClusterState currentState, List jo remoteStoreService.updateClusterStateRepositoriesMetadata(new RemoteStoreNode(node), currentState) ); } + } else if (currentNodes.nodeExistsWithSameRoles(node)) { + logger.debug("received a join request for an existing node [{}]", node); } else { try { if (enforceMajorVersion) { From 7e9536121a89918f66b93a1d14ab0c382512e184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sat, 2 Sep 2023 18:53:13 +0530 Subject: [PATCH 17/34] Fixing all UTs and ITs, Addressing Comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- ...emoteStoreMockRepositoryIntegTestCase.java | 22 ++-- .../remotestore/PrimaryTermValidationIT.java | 1 + .../remotestore/RemoteRestoreSnapshotIT.java | 2 + .../RemoteStoreBaseIntegTestCase.java | 22 ++-- .../remotestore/RemoteStoreForceMergeIT.java | 2 +- .../RemoteStoreRepositoryRegistrationIT.java | 16 ++- .../remotestore/RemoteStoreRestoreIT.java | 78 +++++++++++--- .../ReplicaToPrimaryPromotionIT.java | 4 +- .../cluster/coordination/Coordinator.java | 4 +- .../cluster/coordination/JoinHelper.java | 2 +- .../coordination/JoinTaskExecutor.java | 102 +++++++++++------- .../metadata/MetadataCreateIndexService.java | 25 +++-- .../cluster/node/DiscoveryNode.java | 4 +- .../common/settings/ClusterSettings.java | 2 +- .../opensearch/discovery/DiscoveryModule.java | 2 +- .../main/java/org/opensearch/node/Node.java | 4 +- .../RemoteStoreNodeAttribute.java} | 10 +- .../remotestore/RemoteStoreNodeService.java | 82 +++++++------- .../cluster/coordination/JoinHelperTests.java | 2 +- .../coordination/JoinTaskExecutorTests.java | 45 ++++---- .../cluster/coordination/NodeJoinTests.java | 2 +- .../MetadataCreateIndexServiceTests.java | 4 +- .../discovery/DiscoveryModuleTests.java | 2 +- .../indices/cluster/ClusterStateChanges.java | 2 +- .../BlobStoreRepositoryRemoteIndexTests.java | 8 +- .../snapshots/SnapshotResiliencyTests.java | 2 +- .../AbstractCoordinatorTestCase.java | 2 +- 27 files changed, 274 insertions(+), 179 deletions(-) rename server/src/main/java/org/opensearch/{action/admin/cluster/remotestore/RemoteStoreNode.java => node/remotestore/RemoteStoreNodeAttribute.java} (95%) rename server/src/main/java/org/opensearch/{action/admin/cluster => node}/remotestore/RemoteStoreNodeService.java (65%) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 83d8e282c3474..00763dc2e31fc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -30,11 +30,11 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { @@ -120,13 +120,15 @@ protected String setup(Path repoLocation, double ioFailureRate, String skipExcep // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the // repository creation can happen without failure. - Settings settings = Settings.builder() - .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomBoolean() ? ReplicationType.SEGMENT : ReplicationType.DOCUMENT) - .put(buildRemoteStoreNodeAttributes(repoLocation, ioFailureRate, skipExceptionBlobList, maxFailure)) - .build(); + Settings.Builder settings = Settings.builder() + .put(buildRemoteStoreNodeAttributes(repoLocation, ioFailureRate, skipExceptionBlobList, maxFailure)); + + if (randomBoolean()) { + settings.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT); + } - internalCluster().startClusterManagerOnlyNode(settings); - String dataNodeName = internalCluster().startDataOnlyNode(settings); + internalCluster().startClusterManagerOnlyNode(settings.build()); + String dataNodeName = internalCluster().startDataOnlyNode(settings.build()); // assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME); logger.info("--> Created index={}", INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index 21c8dcfb333ce..e14a4062f7775 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -156,6 +156,7 @@ public void testPrimaryTermValidation() throws Exception { // received the following exception. ShardNotFoundException exception = assertThrows(ShardNotFoundException.class, () -> indexSameDoc(primaryNode, INDEX_NAME)); assertTrue(exception.getMessage().contains("no such shard")); + internalCluster().clearDisruptionScheme(); ensureStableCluster(3); ensureGreen(INDEX_NAME); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 11039742f5d20..33c59c2850388 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -28,6 +28,7 @@ import org.opensearch.snapshots.AbstractSnapshotIntegTestCase; import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; import java.io.IOException; @@ -42,6 +43,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase { private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; private Path remoteRepoPath; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index d2793eaeb3e5f..dbbb2898b4a76 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -39,11 +39,11 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { @@ -58,6 +58,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected Path segmentRepoPath; protected Path translogRepoPath; + protected boolean clusterSettingsSuppliedByTest = false; private final List documentKeys = List.of( randomAlphaOfLength(5), randomAlphaOfLength(5), @@ -115,10 +116,14 @@ protected Settings nodeSettings(int nodeOrdinal) { segmentRepoPath = randomRepoPath().toAbsolutePath(); translogRepoPath = randomRepoPath().toAbsolutePath(); } - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)) - .build(); + if (clusterSettingsSuppliedByTest) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).build(); + } else { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)) + .build(); + } } @Override @@ -251,6 +256,7 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel @After public void teardown() { + clusterSettingsSuppliedByTest = false; assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index 950ef5c3b722c..0bcde4b44c734 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -91,7 +91,7 @@ private void verifyRestoredData(Map indexStats, long deletedDocs) private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) throws IOException { - // internalCluster().startNodes(3); + internalCluster().startNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 47d3c29756cd0..1705f7cec61f2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -24,8 +24,8 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreRepositoryRegistrationIT extends RemoteStoreBaseIntegTestCase { @@ -86,4 +86,16 @@ public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() thro internalCluster().startNodes(3); assertRemoteStoreRepositoryOnAllNodes(); } + + public void testMultiNodeClusterActiveMasterShutDown() throws Exception { + internalCluster().startNodes(3); + internalCluster().stopCurrentClusterManagerNode(); + ensureStableCluster(2); + } + + public void testMultiNodeClusterRandomNodeShutDown() throws Exception { + internalCluster().startNodes(3); + internalCluster().stopRandomDataNode(); + ensureStableCluster(2); + } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index aeb4756c21a8b..595a28761d100 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -16,8 +16,11 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -26,10 +29,13 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThan; @@ -88,8 +94,19 @@ private void verifyRestoredData(Map indexStats, String indexName) } private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { - internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); - internalCluster().startDataOnlyNodes(numDataOnlyNodes); + prepareCluster(numClusterManagerNodes, numDataOnlyNodes, indices, replicaCount, shardCount, Settings.EMPTY); + } + + private void prepareCluster( + int numClusterManagerNodes, + int numDataOnlyNodes, + String indices, + int replicaCount, + int shardCount, + Settings clusterSettings + ) { + internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes, clusterSettings); + internalCluster().startDataOnlyNodes(numDataOnlyNodes, clusterSettings); for (String index : indices.split(",")) { createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); ensureYellowAndNoInitializingShards(index); @@ -452,23 +469,50 @@ public void testRTSRestoreDataOnlyInTranslog() throws Exception { testRestoreFlow(0, true, randomIntBetween(1, 5)); } - public void testRateLimitedRemoteDownloads() throws Exception { - assertAcked( - client().admin() - .cluster() - .preparePutRepository(REPOSITORY_NAME) - .setType("fs") - .setSettings( - Settings.builder() - .put("location", randomRepoPath()) - .put("compress", randomBoolean()) - .put("max_remote_download_bytes_per_sec", "2kb") - .put("chunk_size", 200, ByteSizeUnit.BYTES) - - ) + private Settings buildClusterSettings() { + String segmentRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + String segmentRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); + String translogRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_2_NAME ); + String translogRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_2_NAME + ); + return Settings.builder() + .put( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REPOSITORY_NAME + ) + .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) + .put(segmentRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) + .put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) + .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "2kb") + .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES) + .put( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REPOSITORY_2_NAME + ) + .put(translogRepoTypeAttributeKey, FsRepository.TYPE) + .put(translogRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) + .build(); + } + + public void testRateLimitedRemoteDownloads() throws Exception { + clusterSettingsSuppliedByTest = true; int shardCount = randomIntBetween(1, 3); - prepareCluster(1, 3, INDEX_NAME, 0, shardCount); + prepareCluster(1, 3, INDEX_NAME, 0, shardCount, buildClusterSettings()); Map indexStats = indexData(5, false, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index 3e41c4925ea46..4959a74b75ad4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -@OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class ReplicaToPrimaryPromotionIT extends RemoteStoreBaseIntegTestCase { private int shard_count = 5; @@ -99,7 +99,7 @@ public void testPromoteReplicaToPrimary() throws Exception { final DiscoveryNode randomNode = state.nodes().resolveNode(primaryShard.currentNodeId()); // stop the random data node, all remaining shards are promoted to primaries - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(randomNode.getName())); + internalCluster().stopCurrentClusterManagerNode(); ensureYellowAndNoInitializingShards(indexName); state = client(internalCluster().getClusterManagerName()).admin().cluster().prepareState().get().getState(); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 321b8a65e4d23..4f479122175e8 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -35,7 +35,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -86,6 +85,7 @@ import org.opensearch.discovery.SeedHostsResolver; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool.Names; import org.opensearch.transport.TransportService; @@ -613,8 +613,6 @@ private void handleJoinRequest(JoinRequest joinRequest, JoinHelper.JoinCallback // we are checking source node commission status here to reject any join request coming from a decommissioned node // even before executing the join task to fail fast JoinTaskExecutor.ensureNodeCommissioned(joinRequest.getSourceNode(), stateForJoinValidation.metadata()); - - JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joinRequest.getSourceNode(), stateForJoinValidation); } sendValidateJoinRequest(stateForJoinValidation, joinRequest, joinCallback); } else { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 14f30c272a3a4..b04abb1cd9ebd 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -37,7 +37,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateTaskListener; @@ -62,6 +61,7 @@ import org.opensearch.core.transport.TransportResponse.Empty; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.threadpool.ThreadPool.Names; import org.opensearch.transport.BytesTransportRequest; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index fbec77548aaca..54f511b522b77 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -33,8 +33,6 @@ import org.apache.logging.log4j.Logger; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; import org.opensearch.cluster.NotClusterManagerException; @@ -42,13 +40,17 @@ import org.opensearch.cluster.decommission.NodeDecommissionedException; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.common.Priority; +import org.opensearch.common.SetOnce; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.persistent.PersistentTasksCustomMetadata; import java.util.ArrayList; @@ -61,11 +63,11 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.CompatibilityMode; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT; +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; /** * Main executor for Nodes joining the OpenSearch cluster @@ -174,6 +176,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); + SetOnce repositoriesMetadata = new SetOnce<>(); assert nodesBuilder.isLocalNodeElectedClusterManager(); @@ -187,34 +190,24 @@ public ClusterTasksResult execute(ClusterState currentState, List jo final DiscoveryNode node = joinTask.node(); if (joinTask.isBecomeClusterManagerTask() || joinTask.isFinishElectionTask()) { // noop - } else if (currentNodes.nodeExists(node)) { - // TODO: Fix this by moving it out of this if condition, Had to add this code back here as this was - // leading to failure of JoinTaskExecutorTests::testUpdatesNodeWithNewRoles test. - if (node.isRemoteStoreNode()) { - /** cluster state is updated here as elect leader task can have same node present in join task as - * well as current node. We want the repositories to be added in cluster state during first node - * join. See - * {@link org.opensearch.gateway.GatewayMetaState#prepareInitialClusterState(TransportService, ClusterService, ClusterState)} **/ - newState = ClusterState.builder( - remoteStoreService.updateClusterStateRepositoriesMetadata(new RemoteStoreNode(node), currentState) - ); - } } else if (currentNodes.nodeExistsWithSameRoles(node)) { logger.debug("received a join request for an existing node [{}]", node); + + repositoriesMetadata.trySet( + remoteStoreService.updateRepositoriesMetadata(node, currentState.getMetadata().custom(RepositoriesMetadata.TYPE)) + ); } else { try { if (enforceMajorVersion) { ensureMajorVersionBarrier(node.getVersion(), minClusterNodeVersion); } - ensureNodesCompatibility(node.getVersion(), minClusterNodeVersion, maxClusterNodeVersion); + ensureNodesCompatibility(node, currentNodes, currentState.metadata(), minClusterNodeVersion, maxClusterNodeVersion); // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices // we have to reject nodes that don't support all indices we have in this cluster ensureIndexCompatibility(node.getVersion(), currentState.getMetadata()); // we have added the same check in handleJoinRequest method and adding it here as this method // would guarantee that a decommissioned node would never be able to join the cluster and ensures correctness ensureNodeCommissioned(node, currentState.metadata()); - - ensureRemoteStoreNodesCompatibility(node, currentState); nodesBuilder.add(node); nodesChanged = true; minClusterNodeVersion = Version.min(minClusterNodeVersion, node.getVersion()); @@ -222,12 +215,9 @@ public ClusterTasksResult execute(ClusterState currentState, List jo if (node.isClusterManagerNode()) { joiniedNodeNameIds.put(node.getName(), node.getId()); } - if (node.isRemoteStoreNode()) { - // Try updating repositories metadata in cluster state once its compatible with the cluster. - newState = ClusterState.builder( - remoteStoreService.updateClusterStateRepositoriesMetadata(new RemoteStoreNode(node), currentState) - ); - } + repositoriesMetadata.trySet( + remoteStoreService.updateRepositoriesMetadata(node, currentState.getMetadata().custom(RepositoriesMetadata.TYPE)) + ); } catch (IllegalArgumentException | IllegalStateException | NodeDecommissionedException e) { results.failure(joinTask, e); continue; @@ -266,16 +256,39 @@ public ClusterTasksResult execute(ClusterState currentState, List jo .coordinationMetadata(coordMetadataBuilder.build()) .build(); return results.build( - allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).metadata(newMetadata).build()) + allocationService.adaptAutoExpandReplicas( + newState.nodes(nodesBuilder) + .metadata(updateMetadataWithRepositoriesMetadata(newMetadata, repositoriesMetadata)) + .build() + ) ); } } - return results.build(allocationService.adaptAutoExpandReplicas(newState.nodes(nodesBuilder).build())); + return results.build( + allocationService.adaptAutoExpandReplicas( + newState.nodes(nodesBuilder) + .metadata(updateMetadataWithRepositoriesMetadata(currentState.metadata(), repositoriesMetadata)) + .build() + ) + ); } else { // we must return a new cluster state instance to force publishing. This is important // for the joining node to finalize its join and set us as a cluster-manager - return results.build(newState.build()); + return results.build( + newState.metadata(updateMetadataWithRepositoriesMetadata(currentState.metadata(), repositoriesMetadata)).build() + ); + } + } + + protected Metadata updateMetadataWithRepositoriesMetadata( + Metadata currentMetadata, + SetOnce repositoriesMetadata + ) { + if (repositoriesMetadata == null || repositoriesMetadata.get() == null || repositoriesMetadata.get().repositories().isEmpty()) { + return currentMetadata; + } else { + return Metadata.builder(currentMetadata).putCustom(RepositoriesMetadata.TYPE, repositoriesMetadata.get()).build(); } } @@ -393,16 +406,24 @@ public static void ensureIndexCompatibility(final Version nodeVersion, Metadata /** * ensures that the joining node has a version that's compatible with all current nodes */ - public static void ensureNodesCompatibility(final Version joiningNodeVersion, DiscoveryNodes currentNodes) { + public static void ensureNodesCompatibility(final DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { final Version minNodeVersion = currentNodes.getMinNodeVersion(); final Version maxNodeVersion = currentNodes.getMaxNodeVersion(); - ensureNodesCompatibility(joiningNodeVersion, minNodeVersion, maxNodeVersion); + ensureNodesCompatibility(joiningNode, currentNodes, metadata, minNodeVersion, maxNodeVersion); } /** - * ensures that the joining node has a version that's compatible with a given version range + * ensures that the joining node has a version that's compatible with a given version range and ensures that the + * joining node has required attributes to join a remotestore cluster. */ - public static void ensureNodesCompatibility(Version joiningNodeVersion, Version minClusterNodeVersion, Version maxClusterNodeVersion) { + public static void ensureNodesCompatibility( + DiscoveryNode joiningNode, + DiscoveryNodes currentNodes, + Metadata metadata, + Version minClusterNodeVersion, + Version maxClusterNodeVersion + ) { + Version joiningNodeVersion = joiningNode.getVersion(); assert minClusterNodeVersion.onOrBefore(maxClusterNodeVersion) : minClusterNodeVersion + " > " + maxClusterNodeVersion; if (joiningNodeVersion.isCompatible(maxClusterNodeVersion) == false) { throw new IllegalStateException( @@ -424,6 +445,8 @@ public static void ensureNodesCompatibility(Version joiningNodeVersion, Version + "], which is incompatible." ); } + + ensureRemoteStoreNodesCompatibility(joiningNode, currentNodes, metadata); } /** @@ -470,8 +493,8 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) * TODO: When we support moving from remote store cluster to non remote store and vice versa the this logic will * needs to be modified. */ - public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, ClusterState currentState) { - List existingNodes = new ArrayList<>(currentState.getNodes().getNodes().values()); + private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { + List existingNodes = new ArrayList<>(currentNodes.getNodes().values()); // If there are no node in the cluster state we will No op the compatibility check as at this point we cannot // determine if this is a remote store cluster or non-remote store cluster. @@ -481,13 +504,13 @@ public static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode // TODO: The below check is valid till we support migration, once we start supporting migration a remote // store node will be able to join a non remote store cluster and vice versa. #7986 - CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(currentState.metadata().settings()); + CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); if (STRICT.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { - RemoteStoreNode joiningRemoteStoreNode = new RemoteStoreNode(joiningNode); - RemoteStoreNode existingRemoteStoreNode = new RemoteStoreNode(existingNode); + RemoteStoreNodeAttribute joiningRemoteStoreNode = new RemoteStoreNodeAttribute(joiningNode); + RemoteStoreNodeAttribute existingRemoteStoreNode = new RemoteStoreNodeAttribute(existingNode); if (existingRemoteStoreNode.equals(joiningRemoteStoreNode) == false) { throw new IllegalStateException( "a remote store node [" @@ -518,10 +541,9 @@ public static Collection> addBuiltInJoin ) { final Collection> validators = new ArrayList<>(); validators.add((node, state) -> { - ensureNodesCompatibility(node.getVersion(), state.getNodes()); + ensureNodesCompatibility(node, state.getNodes(), state.metadata()); ensureIndexCompatibility(node.getVersion(), state.getMetadata()); ensureNodeCommissioned(node, state.getMetadata()); - ensureRemoteStoreNodesCompatibility(node, state); }); validators.addAll(onJoinValidators); return Collections.unmodifiableCollection(validators); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 9bef81d9019cb..145eb15e4ebe9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -39,7 +39,6 @@ import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.opensearch.action.admin.indices.shrink.ResizeType; @@ -96,6 +95,7 @@ import org.opensearch.indices.SystemIndices; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -937,12 +937,13 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, settingsBuilder.put(SETTING_REPLICATION_TYPE, INDEX_REPLICATION_TYPE_SETTING.get(requestSettings)); } else if (CLUSTER_REPLICATION_TYPE_SETTING.exists(clusterSettings)) { settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.get(clusterSettings)); - } else if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) - .isEmpty() == false) { - settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); - } else { - settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings)); - } + } else if (clusterSettings.getByPrefix( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX + ).isEmpty() == false) { + settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); + } else { + settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings)); + } } /** @@ -951,16 +952,20 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, * @param clusterSettings cluster level settings */ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings clusterSettings) { - if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) + if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) .isEmpty() == false) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put( SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - clusterSettings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) + clusterSettings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY + ) ) .put( SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, - clusterSettings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY) + clusterSettings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY + ) ); } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 1df6984c5659e..0c8a2e843c6c2 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -33,7 +33,6 @@ package org.opensearch.cluster.node; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; @@ -45,6 +44,7 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import java.io.IOException; import java.util.Collections; @@ -61,8 +61,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; /** * A discovery node represents a node that is part of the cluster. diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 7c5efb8f19df0..1d9ccc8e67bf2 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -33,7 +33,6 @@ import org.apache.logging.log4j.LogManager; import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.indices.close.TransportCloseIndexAction; import org.opensearch.action.search.CreatePitController; import org.opensearch.action.search.TransportSearchAction; @@ -130,6 +129,7 @@ import org.opensearch.node.Node; import org.opensearch.node.Node.DiscoverySettings; import org.opensearch.node.NodeRoleSettings; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.persistent.PersistentTasksClusterService; import org.opensearch.persistent.decider.EnableAssignmentDecider; import org.opensearch.plugins.PluginsService; diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index cead506349169..e581dc029c19c 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -34,7 +34,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.coordination.ElectionStrategy; @@ -54,6 +53,7 @@ import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.gateway.GatewayMetaState; import org.opensearch.monitor.NodeHealthService; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.plugins.DiscoveryPlugin; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 0522a3604db34..3580cd2727860 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -43,7 +43,6 @@ import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; import org.opensearch.action.search.SearchPhaseController; @@ -166,6 +165,7 @@ import org.opensearch.monitor.fs.FsInfo; import org.opensearch.monitor.fs.FsProbe; import org.opensearch.monitor.jvm.JvmInfo; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.persistent.PersistentTasksClusterService; import org.opensearch.persistent.PersistentTasksExecutor; import org.opensearch.persistent.PersistentTasksExecutorRegistry; @@ -267,10 +267,10 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.common.util.FeatureFlags.TELEMETRY; import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; /** * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java similarity index 95% rename from server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java rename to server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index a6bd099679aa4..e61c95edbe5fe 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNode.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.action.admin.cluster.remotestore; +package org.opensearch.node.remotestore; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; @@ -28,7 +28,7 @@ * * @opensearch.internal */ -public class RemoteStoreNode { +public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; @@ -39,9 +39,9 @@ public class RemoteStoreNode { private final DiscoveryNode node; /** - * Creates a new {@link RemoteStoreNode} + * Creates a new {@link RemoteStoreNodeAttribute} */ - public RemoteStoreNode(DiscoveryNode node) { + public RemoteStoreNodeAttribute(DiscoveryNode node) { this.node = node; this.repositoriesMetadata = buildRepositoriesMetadata(); } @@ -119,7 +119,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - RemoteStoreNode that = (RemoteStoreNode) o; + RemoteStoreNodeAttribute that = (RemoteStoreNodeAttribute) o; return this.getRepositoriesMetadata().equalsIgnoreGenerations(that.getRepositoriesMetadata()); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java similarity index 65% rename from server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java rename to server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 6b6733c7f1ab7..654f1f2bd50be 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -6,13 +6,11 @@ * compatible open source license. */ -package org.opensearch.action.admin.cluster.remotestore; +package org.opensearch.node.remotestore; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; @@ -22,7 +20,6 @@ import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.function.Supplier; @@ -85,7 +82,7 @@ public RemoteStoreNodeService(Supplier repositoriesService, * to the repository with appropriate permissions. */ public List createAndVerifyRepositories(DiscoveryNode localNode) { - RemoteStoreNode node = new RemoteStoreNode(localNode); + RemoteStoreNodeAttribute node = new RemoteStoreNodeAttribute(localNode); List repositories = new ArrayList<>(); for (RepositoryMetadata repositoryMetadata : node.getRepositoriesMetadata().repositories()) { String repositoryName = repositoryMetadata.name(); @@ -110,49 +107,48 @@ public List createAndVerifyRepositories(DiscoveryNode localNode) { return repositories; } - private ClusterState updateRepositoryMetadata(RepositoryMetadata newRepositoryMetadata, ClusterState currentState) { - Metadata metadata = currentState.metadata(); - Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata()); - RepositoriesMetadata repositories = metadata.custom(RepositoriesMetadata.TYPE); - if (repositories == null) { - repositories = new RepositoriesMetadata(Collections.singletonList(newRepositoryMetadata)); - } else { - List repositoriesMetadata = new ArrayList<>(repositories.repositories().size() + 1); - - for (RepositoryMetadata repositoryMetadata : repositories.repositories()) { - if (repositoryMetadata.name().equals(newRepositoryMetadata.name())) { - if (newRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)) { - return new ClusterState.Builder(currentState).build(); - } else { - throw new IllegalStateException( - "new repository metadata [" - + newRepositoryMetadata - + "] supplied by joining node is different from existing repository metadata [" - + repositoryMetadata - + "]." - ); - } - } else { - repositoriesMetadata.add(repositoryMetadata); - } - } - repositoriesMetadata.add(newRepositoryMetadata); - repositories = new RepositoriesMetadata(repositoriesMetadata); - } - mdBuilder.putCustom(RepositoriesMetadata.TYPE, repositories); - return ClusterState.builder(currentState).metadata(mdBuilder).build(); - } - /** * Updates repositories metadata in the cluster state if not already present. If a repository metadata for a * repository is already present in the cluster state and if it's different then the joining remote store backed * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ - public ClusterState updateClusterStateRepositoriesMetadata(RemoteStoreNode joiningNode, ClusterState currentState) { - ClusterState newState = ClusterState.builder(currentState).build(); - for (RepositoryMetadata newRepositoryMetadata : joiningNode.getRepositoriesMetadata().repositories()) { - newState = updateRepositoryMetadata(newRepositoryMetadata, newState); + public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode, RepositoriesMetadata existingRepositories) { + if (joiningNode.isRemoteStoreNode()) { + List updatedRepositoryMetadataList = new ArrayList<>(); + List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode).getRepositoriesMetadata() + .repositories(); + + if (existingRepositories == null) { + return new RepositoriesMetadata(newRepositoryMetadataList); + } else { + updatedRepositoryMetadataList.addAll(existingRepositories.repositories()); + } + + for (RepositoryMetadata newRepositoryMetadata : newRepositoryMetadataList) { + boolean repositoryAlreadyPresent = false; + for (RepositoryMetadata existingRepositoryMetadata : existingRepositories.repositories()) { + if (newRepositoryMetadata.name().equals(existingRepositoryMetadata.name())) { + if (newRepositoryMetadata.equalsIgnoreGenerations(existingRepositoryMetadata)) { + repositoryAlreadyPresent = true; + break; + } else { + throw new IllegalStateException( + "new repository metadata [" + + newRepositoryMetadata + + "] supplied by joining node is different from existing repository metadata [" + + existingRepositoryMetadata + + "]." + ); + } + } + } + if (repositoryAlreadyPresent == false) { + updatedRepositoryMetadataList.add(newRepositoryMetadata); + } + } + return new RepositoriesMetadata(updatedRepositoryMetadataList); + } else { + return existingRepositories; } - return newState; } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java index 5d6c632b3a499..c6f3ab75b9cce 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java @@ -34,7 +34,6 @@ import org.apache.logging.log4j.Level; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -49,6 +48,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.transport.TransportResponse; import org.opensearch.monitor.StatusInfo; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index fc66f9bd45dda..211b5eccb748e 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -33,7 +33,6 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; @@ -53,6 +52,7 @@ import org.opensearch.common.SetOnce; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; @@ -67,10 +67,10 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.VersionUtils.allVersions; import static org.opensearch.test.VersionUtils.maxCompatibleVersion; import static org.opensearch.test.VersionUtils.randomCompatibleVersion; @@ -123,14 +123,20 @@ public void testPreventJoinClusterWithUnsupportedNodeVersions() { builder.add(new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), randomCompatibleVersion(random(), version))); DiscoveryNodes nodes = builder.build(); + Metadata metadata = Metadata.EMPTY_METADATA; + final Version maxNodeVersion = nodes.getMaxNodeVersion(); final Version minNodeVersion = nodes.getMinNodeVersion(); - final Version tooLow = LegacyESVersion.fromString("6.7.0"); + final DiscoveryNode tooLowJoiningNode = new DiscoveryNode( + UUIDs.base64UUID(), + buildNewFakeTransportAddress(), + LegacyESVersion.fromString("6.7.0") + ); expectThrows(IllegalStateException.class, () -> { if (randomBoolean()) { - JoinTaskExecutor.ensureNodesCompatibility(tooLow, nodes); + JoinTaskExecutor.ensureNodesCompatibility(tooLowJoiningNode, nodes, metadata); } else { - JoinTaskExecutor.ensureNodesCompatibility(tooLow, minNodeVersion, maxNodeVersion); + JoinTaskExecutor.ensureNodesCompatibility(tooLowJoiningNode, nodes, metadata, minNodeVersion, maxNodeVersion); } }); @@ -148,11 +154,11 @@ public void testPreventJoinClusterWithUnsupportedNodeVersions() { minGoodVersion = minCompatVersion.before(allVersions().get(0)) ? allVersions().get(0) : minCompatVersion; } final Version justGood = randomVersionBetween(random(), minGoodVersion, maxCompatibleVersion(minNodeVersion)); - + final DiscoveryNode justGoodJoiningNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), justGood); if (randomBoolean()) { - JoinTaskExecutor.ensureNodesCompatibility(justGood, nodes); + JoinTaskExecutor.ensureNodesCompatibility(justGoodJoiningNode, nodes, metadata); } else { - JoinTaskExecutor.ensureNodesCompatibility(justGood, minNodeVersion, maxNodeVersion); + JoinTaskExecutor.ensureNodesCompatibility(justGoodJoiningNode, nodes, metadata, minNodeVersion, maxNodeVersion); } } @@ -189,6 +195,7 @@ public void testUpdatesNodeWithNewRoles() throws Exception { when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); final RemoteStoreNodeService remoteStoreService = mock(RemoteStoreNodeService.class); + when(remoteStoreService.updateRepositoriesMetadata(any(), any())).thenReturn(new RepositoriesMetadata(Collections.emptyList())); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -359,14 +366,14 @@ public void testJoinClusterWithNonRemoteStoreNodeJoining() { ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).build(); DiscoveryNode joiningNode = newDiscoveryNode(Collections.emptyMap()); - JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } public void testJoinClusterWithRemoteStoreNodeJoining() { ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).build(); DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); - JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } public void testJoinClusterWithNonRemoteStoreNodeJoiningNonRemoteStoreCluster() { @@ -377,7 +384,7 @@ public void testJoinClusterWithNonRemoteStoreNodeJoiningNonRemoteStoreCluster() DiscoveryNode joiningNode = newDiscoveryNode(Collections.emptyMap()); - JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } public void testPreventJoinClusterWithRemoteStoreNodeJoiningNonRemoteStoreCluster() { @@ -389,7 +396,7 @@ public void testPreventJoinClusterWithRemoteStoreNodeJoiningNonRemoteStoreCluste DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); Exception e = assertThrows( IllegalStateException.class, - () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) ); assertTrue(e.getMessage().equals("a remote store node [" + joiningNode + "] is trying to join a non remote " + "store cluster")); } @@ -407,7 +414,7 @@ public void testJoinClusterWithRemoteStoreNodeJoiningRemoteStoreCluster() { .build(); DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); - JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentAttributesJoiningRemoteStoreCluster() { @@ -473,7 +480,7 @@ public void testPreventJoinClusterWithNonRemoteStoreNodeJoiningRemoteStoreCluste DiscoveryNode joiningNode = newDiscoveryNode(Collections.emptyMap()); Exception e = assertThrows( IllegalStateException.class, - () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) ); assertTrue(e.getMessage().equals("a non remote store node [" + joiningNode + "] is trying to join a remote " + "store cluster")); } @@ -497,7 +504,7 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithPartialAttributesJoinin DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); Exception e = assertThrows( IllegalStateException.class, - () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) ); assertTrue( e.getMessage().equals("joining node [" + joiningNode + "] doesn't have the node attribute [" + nodeAttribute.getKey() + "]") @@ -799,7 +806,7 @@ private void validateAttributes(Map remoteStoreNodeAttributes, C DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); Exception e = assertThrows( IllegalStateException.class, - () -> JoinTaskExecutor.ensureRemoteStoreNodesCompatibility(joiningNode, currentState) + () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()) ); assertTrue( e.getMessage() diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java index bd1e6d9b9ec71..766a20fda8d28 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java @@ -33,7 +33,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.OpenSearchAllocationTestCase; @@ -61,6 +60,7 @@ import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.test.ClusterServiceUtils; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.CapturingTransport; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 820836f41249d..86e154c547e07 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -114,8 +114,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK; @@ -141,6 +139,8 @@ import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.opensearch.node.Node.NODE_ATTRIBUTES; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index 8cd742d5104ea..014fc59e8e76d 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -32,7 +32,6 @@ package org.opensearch.discovery; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.coordination.PersistedStateRegistry; @@ -47,6 +46,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.gateway.GatewayMetaState; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.plugins.DiscoveryPlugin; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.transport.MockTransportService; diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 31bf54e9bcc0c..11721a4b831bd 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -36,7 +36,6 @@ import org.apache.logging.log4j.Logger; import org.opensearch.ExceptionsHelper; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest; import org.opensearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.opensearch.action.admin.indices.close.CloseIndexRequest; @@ -109,6 +108,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.ShardLimitValidator; import org.opensearch.indices.SystemIndices; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.snapshots.EmptySnapshotsInfoService; import org.opensearch.test.gateway.TestGatewayAllocator; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index d59a33b4cbbdd..39900e44856ee 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -58,11 +58,11 @@ import java.util.Locale; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.equalTo; /** diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 59ebbf7204b8a..289499e95e639 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -40,7 +40,6 @@ import org.opensearch.action.ActionType; import org.opensearch.action.RequestValidators; import org.opensearch.action.StepListener; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -206,6 +205,7 @@ import org.opensearch.ingest.IngestService; import org.opensearch.monitor.StatusInfo; import org.opensearch.node.ResponseCollectorService; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.plugins.PluginsService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 478dd8d3f43de..db3ee98f701cf 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -39,7 +39,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskListener; From f26b1a2324a352204d81a917d82ff584fdd6ffff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sat, 2 Sep 2023 19:08:23 +0530 Subject: [PATCH 18/34] Removing unintended changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- distribution/src/config/opensearch.yml | 4 ++-- .../AbstractRemoteStoreMockRepositoryIntegTestCase.java | 4 +--- .../admin/cluster => node}/remotestore/package-info.java | 2 +- .../cluster/coordination/AbstractCoordinatorTestCase.java | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) rename server/src/main/java/org/opensearch/{action/admin/cluster => node}/remotestore/package-info.java (81%) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 07d9b278f6eb9..de2d0e023a200 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -92,10 +92,10 @@ ${path.logs} # cluster.remote_store.enabled: true # # Repository to use for segment upload while enforcing remote store for an index -# node.attr.remote_store.segment.repository : my-repo-1 +# node.attr.remote_store.segment.repository: my-repo-1 # # Repository to use for translog upload while enforcing remote store for an index -# node.attr.remote_store.translog.repository : my-repo-1 +# node.attr.remote_store.translog.repository: my-repo-1 # # ---------------------------------- Experimental Features ----------------------------------- # diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 00763dc2e31fc..5913b1c45c037 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -114,9 +114,7 @@ protected void deleteRepo() { assertAcked(clusterAdmin().prepareDeleteRepository(TRANSLOG_REPOSITORY_NAME)); } - protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) throws Exception { - logger.info("--> Creating repository={} at the path={}", REPOSITORY_NAME, repoLocation); - logger.info("--> Creating repository={} at the path={}", TRANSLOG_REPOSITORY_NAME, repoLocation); + protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the // repository creation can happen without failure. diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/package-info.java b/server/src/main/java/org/opensearch/node/remotestore/package-info.java similarity index 81% rename from server/src/main/java/org/opensearch/action/admin/cluster/remotestore/package-info.java rename to server/src/main/java/org/opensearch/node/remotestore/package-info.java index adb42ffa4032f..e2592aa5fcc29 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/package-info.java +++ b/server/src/main/java/org/opensearch/node/remotestore/package-info.java @@ -7,4 +7,4 @@ */ /** Restore remote store transport handler. */ -package org.opensearch.action.admin.cluster.remotestore; +package org.opensearch.node.remotestore; diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index db3ee98f701cf..d9a5a2e1f3fdd 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -39,7 +39,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; import org.opensearch.Version; -import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskListener; @@ -87,6 +86,7 @@ import org.opensearch.gateway.PersistedClusterStateService; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; +import org.opensearch.node.remotestore.RemoteStoreNodeService; import org.opensearch.repositories.RepositoriesService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.disruption.DisruptableMockTransport; From f1785f101b5fd024897114f9c4adb7f2181fa5a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 3 Sep 2023 00:35:05 +0530 Subject: [PATCH 19/34] Fixing setup issues and assertions for a few ITs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteStoreRepositoryRegistrationIT.java | 4 +++- .../java/org/opensearch/snapshots/CloneSnapshotIT.java | 2 +- .../snapshots/RemoteIndexSnapshotStatusApiIT.java | 8 ++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 1705f7cec61f2..98019d7c13c11 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -53,7 +53,7 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na return new RepositoryMetadata(name, type, settings.build()); } - private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { + private void assertRemoteStoreRepositoryOnAllNodes() { RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) .state() .metadata() @@ -91,11 +91,13 @@ public void testMultiNodeClusterActiveMasterShutDown() throws Exception { internalCluster().startNodes(3); internalCluster().stopCurrentClusterManagerNode(); ensureStableCluster(2); + assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRandomNodeShutDown() throws Exception { internalCluster().startNodes(3); internalCluster().stopRandomDataNode(); ensureStableCluster(2); + assertRemoteStoreRepositoryOnAllNodes(); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 55ce2805fcb94..9ec8cee2685fe 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -284,7 +284,7 @@ public void testCloneAfterRepoShallowSettingDisabled() throws Exception { final String remoteStoreRepoName = "remote-store-repo-name"; final Path remoteStoreRepoPath = randomRepoPath(); internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java index 231e8ad3788be..979df95547d06 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java @@ -51,7 +51,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteIndexSnapshotStatusApiIT extends AbstractSnapshotIntegTestCase { protected Path absolutePath; @@ -76,7 +76,7 @@ protected Settings nodeSettings(int nodeOrdinal) { public void testStatusAPICallForShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used for the test"); internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNodes(2); final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); @@ -111,7 +111,7 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception { public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used for the test"); internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNodes(2); final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); @@ -155,7 +155,7 @@ public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { public void testStatusAPICallInProgressShallowSnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used for the test"); internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNodes(2); final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy().put("block_on_data", true)); From 4e05540ed99ba553866e30b3f0a109160ef32069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Sun, 3 Sep 2023 01:08:21 +0530 Subject: [PATCH 20/34] Reverting deleted settings and fixing tests post rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../org/opensearch/remotestore/RemoteStoreIT.java | 7 +++---- .../opensearch/common/settings/ClusterSettings.java | 2 ++ .../java/org/opensearch/indices/IndicesService.java | 11 +++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 345b8369befef..bd019693f01ff 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -196,7 +196,7 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { * default. */ public void testDefaultBufferInterval() throws ExecutionException, InterruptedException { - setupRepo(); + internalCluster().startClusterManagerOnlyNode(); String clusterManagerName = internalCluster().getClusterManagerName(); String dataNode = internalCluster().startDataOnlyNodes(1).get(0); createIndex(INDEX_NAME); @@ -224,7 +224,7 @@ public void testDefaultBufferInterval() throws ExecutionException, InterruptedEx * with and without cluster default. */ public void testOverriddenBufferInterval() throws ExecutionException, InterruptedException { - setupRepo(); + internalCluster().startClusterManagerOnlyNode(); String clusterManagerName = internalCluster().getClusterManagerName(); String dataNode = internalCluster().startDataOnlyNodes(1).get(0); @@ -281,7 +281,7 @@ public void testOverriddenBufferInterval() throws ExecutionException, Interrupte * This tests validation which kicks in during index creation failing creation if the value is less than minimum allowed value. */ public void testOverriddenBufferIntervalValidation() { - setupRepo(); + internalCluster().startClusterManagerOnlyNode(); TimeValue bufferInterval = TimeValue.timeValueSeconds(-1); Settings indexSettings = Settings.builder() .put(indexSettings()) @@ -302,7 +302,6 @@ public void testOverriddenBufferIntervalValidation() { */ public void testClusterBufferIntervalValidation() { String clusterManagerName = internalCluster().startClusterManagerOnlyNode(); - setupRepo(false); IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, () -> client(clusterManagerName).admin() diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 1d9ccc8e67bf2..b8fe322234140 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -684,6 +684,8 @@ public void apply(Settings value, Settings current, Settings previous) { * setting should be moved to {@link #BUILT_IN_CLUSTER_SETTINGS}. */ public static final Map, List> FEATURE_FLAGGED_CLUSTER_SETTINGS = Map.of( + List.of(FeatureFlags.REMOTE_STORE), + List.of(IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING), List.of(FeatureFlags.CONCURRENT_SEGMENT_SEARCH), List.of( SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING, diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 3507c485e5427..2e2a0762ea489 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -243,6 +243,17 @@ public class IndicesService extends AbstractLifecycleComponent Property.Final ); + /** + * Used to specify the default translog buffer interval for remote store backed indexes. + */ + public static final Setting CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING = Setting.timeSetting( + "cluster.remote_store.translog.buffer_interval", + IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, + IndexSettings.MINIMUM_REMOTE_TRANSLOG_BUFFER_INTERVAL, + Property.NodeScope, + Property.Dynamic + ); + /** * This setting is used to set the refresh interval when the {@code index.refresh_interval} index setting is not * provided during index creation or when the existing {@code index.refresh_interval} index setting is set as null. From 7de82b49f91220a71c9c445d7d6889cbfba159bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 4 Sep 2023 00:53:22 +0530 Subject: [PATCH 21/34] Addressing Comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- ...emoteStoreMockRepositoryIntegTestCase.java | 1 - .../RemoteStoreBaseIntegTestCase.java | 43 +++++++ .../RemoteStoreRepositoryRegistrationIT.java | 108 +++++++++--------- .../cluster/coordination/Coordinator.java | 8 +- .../cluster/coordination/JoinHelper.java | 4 +- .../coordination/JoinTaskExecutor.java | 41 ++++--- .../cluster/node/DiscoveryNode.java | 4 +- .../opensearch/discovery/DiscoveryModule.java | 6 +- .../main/java/org/opensearch/node/Node.java | 14 +-- .../remotestore/RemoteStoreNodeService.java | 4 +- .../cluster/coordination/JoinHelperTests.java | 8 +- .../coordination/JoinTaskExecutorTests.java | 26 ++--- .../discovery/DiscoveryModuleTests.java | 6 +- .../indices/cluster/ClusterStateChanges.java | 6 +- .../snapshots/SnapshotResiliencyTests.java | 6 +- .../AbstractCoordinatorTestCase.java | 6 +- 16 files changed, 169 insertions(+), 122 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 5913b1c45c037..73f5278c175a2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -127,7 +127,6 @@ protected String setup(Path repoLocation, double ioFailureRate, String skipExcep internalCluster().startClusterManagerOnlyNode(settings.build()); String dataNodeName = internalCluster().startDataOnlyNode(settings.build()); - // assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME); logger.info("--> Created index={}", INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index dbbb2898b4a76..d1d2cebc9bed2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -16,6 +16,10 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -23,6 +27,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; @@ -38,6 +43,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; @@ -257,10 +263,47 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel @After public void teardown() { clusterSettingsSuppliedByTest = false; + assertRemoteStoreRepositoryOnAllNodes(); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); } + private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + Map nodeAttributes = node.getAttributes(); + String type = nodeAttributes.get(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); + + String settingsAttributeKeyPrefix = String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, name); + Map settingsMap = node.getAttributes() + .keySet() + .stream() + .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> node.getAttributes().get(key))); + + Settings.Builder settings = Settings.builder(); + settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); + + return new RepositoryMetadata(name, type, settings.build()); + } + + private void assertRemoteStoreRepositoryOnAllNodes() { + RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) + .state() + .metadata() + .custom(RepositoriesMetadata.TYPE); + RepositoryMetadata actualSegmentRepository = repositories.repository(REPOSITORY_NAME); + RepositoryMetadata actualTranslogRepository = repositories.repository(REPOSITORY_2_NAME); + + for (String nodeName : internalCluster().getNodeNames()) { + ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); + DiscoveryNode node = clusterService.localNode(); + RepositoryMetadata expectedSegmentRepository = buildRepositoryMetadata(node, REPOSITORY_NAME); + RepositoryMetadata expectedTranslogRepository = buildRepositoryMetadata(node, REPOSITORY_2_NAME); + assertTrue(actualSegmentRepository.equalsIgnoreGenerations(expectedSegmentRepository)); + assertTrue(actualTranslogRepository.equalsIgnoreGenerations(expectedTranslogRepository)); + } + } + public static int getFileCount(Path path) throws Exception { final AtomicInteger filesExisting = new AtomicInteger(0); Files.walkFileTree(path, new SimpleFileVisitor<>() { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 98019d7c13c11..4d56a1e94e3fc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -8,25 +8,17 @@ package org.opensearch.remotestore; -import org.opensearch.cluster.metadata.RepositoriesMetadata; -import org.opensearch.cluster.metadata.RepositoryMetadata; -import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; -import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.disruption.NetworkDisruption; import org.opensearch.test.transport.MockTransportService; import java.util.Arrays; import java.util.Collection; -import java.util.Locale; -import java.util.Map; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; - @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoteStoreRepositoryRegistrationIT extends RemoteStoreBaseIntegTestCase { @@ -35,69 +27,71 @@ protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } - private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { - Map nodeAttributes = node.getAttributes(); - String type = nodeAttributes.get(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); - - String settingsAttributeKeyPrefix = String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, name); - Map settingsMap = node.getAttributes() - .keySet() - .stream() - .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) - .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> node.getAttributes().get(key))); - - Settings.Builder settings = Settings.builder(); - settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); - settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); - - return new RepositoryMetadata(name, type, settings.build()); - } - - private void assertRemoteStoreRepositoryOnAllNodes() { - RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) - .state() - .metadata() - .custom(RepositoriesMetadata.TYPE); - RepositoryMetadata actualSegmentRepository = repositories.repository(REPOSITORY_NAME); - RepositoryMetadata actualTranslogRepository = repositories.repository(REPOSITORY_2_NAME); - - for (String nodeName : internalCluster().getNodeNames()) { - ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); - DiscoveryNode node = clusterService.localNode(); - RepositoryMetadata expectedSegmentRepository = buildRepositoryMetadata(node, REPOSITORY_NAME); - RepositoryMetadata expectedTranslogRepository = buildRepositoryMetadata(node, REPOSITORY_2_NAME); - assertTrue(actualSegmentRepository.equalsIgnoreGenerations(expectedSegmentRepository)); - assertTrue(actualTranslogRepository.equalsIgnoreGenerations(expectedTranslogRepository)); - } - } - public void testSingleNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNode(); - assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNodes(3); - assertRemoteStoreRepositoryOnAllNodes(); } - public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() throws Exception { + public void testMultiNodeClusterRepositoryRegistrationWithMultipleClusterManager() throws Exception { internalCluster().startClusterManagerOnlyNodes(3); internalCluster().startNodes(3); - assertRemoteStoreRepositoryOnAllNodes(); } - public void testMultiNodeClusterActiveMasterShutDown() throws Exception { + public void testMultiNodeClusterActiveClusterManagerShutDown() throws Exception { internalCluster().startNodes(3); internalCluster().stopCurrentClusterManagerNode(); ensureStableCluster(2); - assertRemoteStoreRepositoryOnAllNodes(); } - public void testMultiNodeClusterRandomNodeShutDown() throws Exception { + public void testMultiNodeClusterActiveMClusterManagerRestart() throws Exception { internalCluster().startNodes(3); - internalCluster().stopRandomDataNode(); - ensureStableCluster(2); - assertRemoteStoreRepositoryOnAllNodes(); + String clusterManagerNodeName = internalCluster().getClusterManagerName(); + internalCluster().restartNode(clusterManagerNodeName); + ensureStableCluster(3); + } + + public void testMultiNodeClusterRandomNodeRestart() throws Exception { + internalCluster().startNodes(3); + internalCluster().restartRandomDataNode(); + ensureStableCluster(3); + } + + public void testMultiNodeClusterActiveClusterManagerRecoverNetworkIsolation() { + internalCluster().startClusterManagerOnlyNodes(3); + String dataNode = internalCluster().startNode(); + + NetworkDisruption partition = isolateClusterManagerDisruption(NetworkDisruption.DISCONNECT); + internalCluster().setDisruptionScheme(partition); + + partition.startDisrupting(); + ensureStableCluster(3, dataNode); + partition.stopDisrupting(); + + ensureStableCluster(4); + + internalCluster().clearDisruptionScheme(); + } + + public void testMultiNodeClusterRandomNodeRecoverNetworkIsolation() { + Set nodesInOneSide = internalCluster().startNodes(3).stream().collect(Collectors.toCollection(HashSet::new)); + Set nodesInAnotherSide = internalCluster().startNodes(3).stream().collect(Collectors.toCollection(HashSet::new)); + ensureStableCluster(6); + + NetworkDisruption networkDisruption = new NetworkDisruption( + new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInAnotherSide), + NetworkDisruption.DISCONNECT + ); + internalCluster().setDisruptionScheme(networkDisruption); + + networkDisruption.startDisrupting(); + ensureStableCluster(3, nodesInOneSide.stream().findAny().get()); + networkDisruption.stopDisrupting(); + + ensureStableCluster(6); + + internalCluster().clearDisruptionScheme(); } } diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 4f479122175e8..eb30460ca1b7f 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -183,7 +183,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private Optional currentPublication = Optional.empty(); private final NodeHealthService nodeHealthService; private final PersistedStateRegistry persistedStateRegistry; - private final RemoteStoreNodeService remoteStoreService; + private final RemoteStoreNodeService remoteStoreNodeService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -206,7 +206,7 @@ public Coordinator( ElectionStrategy electionStrategy, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreNodeService remoteStoreService + RemoteStoreNodeService remoteStoreNodeService ) { this.settings = settings; this.transportService = transportService; @@ -220,7 +220,7 @@ public Coordinator( allocationService, clusterManagerService, transportService, - remoteStoreService, + remoteStoreNodeService, this::getCurrentTerm, this::getStateForClusterManagerService, this::handleJoinRequest, @@ -294,7 +294,7 @@ public Coordinator( this.nodeHealthService = nodeHealthService; this.persistedStateRegistry = persistedStateRegistry; this.localNodeCommissioned = true; - this.remoteStoreService = remoteStoreService; + this.remoteStoreNodeService = remoteStoreNodeService; } private ClusterFormationState getClusterFormationState() { diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index b04abb1cd9ebd..9bf6bac07da53 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -136,7 +136,7 @@ public class JoinHelper { AllocationService allocationService, ClusterManagerService clusterManagerService, TransportService transportService, - RemoteStoreNodeService remoteStoreService, + RemoteStoreNodeService remoteStoreNodeService, LongSupplier currentTermSupplier, Supplier currentStateSupplier, BiConsumer joinHandler, @@ -159,7 +159,7 @@ public class JoinHelper { allocationService, logger, rerouteService, - remoteStoreService + remoteStoreNodeService ) { private final long term = currentTermSupplier.getAsLong(); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 54f511b522b77..65155002c2108 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -81,7 +81,7 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor execute(ClusterState currentState, List jo } else if (currentNodes.nodeExistsWithSameRoles(node)) { logger.debug("received a join request for an existing node [{}]", node); - repositoriesMetadata.trySet( - remoteStoreService.updateRepositoriesMetadata(node, currentState.getMetadata().custom(RepositoriesMetadata.TYPE)) - ); + if (repositoriesMetadata.get() == null) { + repositoriesMetadata.trySet( + remoteStoreNodeService.updateRepositoriesMetadata( + node, + currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + ) + ); + } } else { try { if (enforceMajorVersion) { @@ -215,9 +220,14 @@ public ClusterTasksResult execute(ClusterState currentState, List jo if (node.isClusterManagerNode()) { joiniedNodeNameIds.put(node.getName(), node.getId()); } - repositoriesMetadata.trySet( - remoteStoreService.updateRepositoriesMetadata(node, currentState.getMetadata().custom(RepositoriesMetadata.TYPE)) - ); + if (repositoriesMetadata.get() == null) { + repositoriesMetadata.trySet( + remoteStoreNodeService.updateRepositoriesMetadata( + node, + currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + ) + ); + } } catch (IllegalArgumentException | IllegalStateException | NodeDecommissionedException e) { results.failure(joinTask, e); continue; @@ -281,10 +291,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } } - protected Metadata updateMetadataWithRepositoriesMetadata( - Metadata currentMetadata, - SetOnce repositoriesMetadata - ) { + private Metadata updateMetadataWithRepositoriesMetadata(Metadata currentMetadata, SetOnce repositoriesMetadata) { if (repositoriesMetadata == null || repositoriesMetadata.get() == null || repositoriesMetadata.get().repositories().isEmpty()) { return currentMetadata; } else { @@ -502,16 +509,16 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod return; } - // TODO: The below check is valid till we support migration, once we start supporting migration a remote + // TODO: The below check is valid till we don't support migration, once we start supporting migration a remote // store node will be able to join a non remote store cluster and vice versa. #7986 CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); if (STRICT.equals(remoteStoreCompatibilityMode)) { DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { - RemoteStoreNodeAttribute joiningRemoteStoreNode = new RemoteStoreNodeAttribute(joiningNode); - RemoteStoreNodeAttribute existingRemoteStoreNode = new RemoteStoreNodeAttribute(existingNode); - if (existingRemoteStoreNode.equals(joiningRemoteStoreNode) == false) { + RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); + RemoteStoreNodeAttribute existingRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(existingNode); + if (existingRemoteStoreNodeAttribute.equals(joiningRemoteStoreNodeAttribute) == false) { throw new IllegalStateException( "a remote store node [" + joiningNode diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 0c8a2e843c6c2..4e49b25eb5789 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -288,7 +288,7 @@ public static DiscoveryNode createRemoteNodeLocal( Settings settings, TransportAddress publishAddress, String nodeId, - RemoteStoreNodeService remoteStoreService + RemoteStoreNodeService remoteStoreNodeService ) { Map attributes = Node.NODE_ATTRIBUTES.getAsMap(settings); Set roles = getRolesFromSettings(settings); @@ -300,7 +300,7 @@ public static DiscoveryNode createRemoteNodeLocal( roles, Version.CURRENT ); - remoteStoreService.createAndVerifyRepositories(discoveryNode); + remoteStoreNodeService.createAndVerifyRepositories(discoveryNode); return discoveryNode; } diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index e581dc029c19c..4c60562e0b7dc 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -133,7 +133,7 @@ public DiscoveryModule( RerouteService rerouteService, NodeHealthService nodeHealthService, PersistedStateRegistry persistedStateRegistry, - RemoteStoreNodeService remoteStoreService + RemoteStoreNodeService remoteStoreNodeService ) { final Collection> joinValidators = new ArrayList<>(); final Map> hostProviders = new HashMap<>(); @@ -210,8 +210,12 @@ public DiscoveryModule( rerouteService, electionStrategy, nodeHealthService, +<<<<<<< HEAD persistedStateRegistry, remoteStoreService +======= + remoteStoreNodeService +>>>>>>> 5e3b618bf6b (Addressing Comments) ); } else { throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]"); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 3580cd2727860..0a239243256ff 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -531,8 +531,8 @@ protected Node( final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, executorBuilders.toArray(new ExecutorBuilder[0])); final SetOnce repositoriesServiceReference = new SetOnce<>(); - final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService(repositoriesServiceReference::get, threadPool); - localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId(), remoteStoreService); + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService(repositoriesServiceReference::get, threadPool); + localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId(), remoteStoreNodeService); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); resourcesToClose.add(resourceWatcherService); @@ -1005,7 +1005,7 @@ protected Node( rerouteService, fsHealthService, persistedStateRegistry, - remoteStoreService + remoteStoreNodeService ); final SearchPipelineService searchPipelineService = new SearchPipelineService( clusterService, @@ -1744,12 +1744,12 @@ private static class LocalNodeFactory implements Function localNode = new SetOnce<>(); private final String persistentNodeId; private final Settings settings; - private final RemoteStoreNodeService remoteStoreService; + private final RemoteStoreNodeService remoteStoreNodeService; - private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStoreNodeService remoteStoreService) { + private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStoreNodeService remoteStoreNodeService) { this.persistentNodeId = persistentNodeId; this.settings = settings; - this.remoteStoreService = remoteStoreService; + this.remoteStoreNodeService = remoteStoreNodeService; } @Override @@ -1763,7 +1763,7 @@ public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { settings, boundTransportAddress.publishAddress(), persistentNodeId, - remoteStoreService + remoteStoreNodeService ) ); } else { diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 654f1f2bd50be..46aaca6e9cf59 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -82,9 +82,9 @@ public RemoteStoreNodeService(Supplier repositoriesService, * to the repository with appropriate permissions. */ public List createAndVerifyRepositories(DiscoveryNode localNode) { - RemoteStoreNodeAttribute node = new RemoteStoreNodeAttribute(localNode); + RemoteStoreNodeAttribute nodeAttribute = new RemoteStoreNodeAttribute(localNode); List repositories = new ArrayList<>(); - for (RepositoryMetadata repositoryMetadata : node.getRepositoriesMetadata().repositories()) { + for (RepositoryMetadata repositoryMetadata : nodeAttribute.getRepositoriesMetadata().repositories()) { String repositoryName = repositoryMetadata.name(); // Create Repository diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java index c6f3ab75b9cce..3fa5768f4f614 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java @@ -103,7 +103,7 @@ public void testJoinDeduplication() { null, null, transportService, - buildRemoteStoreService(transportService, deterministicTaskQueue.getThreadPool()), + buildRemoteStoreNodeService(transportService, deterministicTaskQueue.getThreadPool()), () -> 0L, () -> null, (joinRequest, joinCallback) -> { @@ -289,7 +289,7 @@ public void testJoinFailureOnUnhealthyNodes() { null, null, transportService, - buildRemoteStoreService(transportService, deterministicTaskQueue.getThreadPool()), + buildRemoteStoreNodeService(transportService, deterministicTaskQueue.getThreadPool()), () -> 0L, () -> null, (joinRequest, joinCallback) -> { @@ -489,7 +489,7 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin null, null, transportService, - buildRemoteStoreService(transportService, deterministicTaskQueue.getThreadPool()), + buildRemoteStoreNodeService(transportService, deterministicTaskQueue.getThreadPool()), () -> 0L, () -> localClusterState, (joinRequest, joinCallback) -> { @@ -509,7 +509,7 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin return new TestClusterSetup(deterministicTaskQueue, localNode, transportService, localClusterState, joinHelper, capturingTransport); } - private RemoteStoreNodeService buildRemoteStoreService(TransportService transportService, ThreadPool threadPool) { + private RemoteStoreNodeService buildRemoteStoreNodeService(TransportService transportService, ThreadPool threadPool) { RepositoriesService repositoriesService = new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 211b5eccb748e..38cd0a1cf6843 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -194,15 +194,15 @@ public void testUpdatesNodeWithNewRoles() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreNodeService remoteStoreService = mock(RemoteStoreNodeService.class); - when(remoteStoreService.updateRepositoriesMetadata(any(), any())).thenReturn(new RepositoriesMetadata(Collections.emptyList())); + final RemoteStoreNodeService remoteStoreNodeService = mock(RemoteStoreNodeService.class); + when(remoteStoreNodeService.updateRepositoriesMetadata(any(), any())).thenReturn(new RepositoriesMetadata(Collections.emptyList())); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, allocationService, logger, rerouteService, - remoteStoreService + remoteStoreNodeService ); final DiscoveryNode clusterManagerNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); @@ -299,14 +299,14 @@ public void testJoinFailedForDecommissionedNode() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreNodeService remoteStoreService = mock(RemoteStoreNodeService.class); + final RemoteStoreNodeService remoteStoreNodeService = mock(RemoteStoreNodeService.class); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, allocationService, logger, rerouteService, - remoteStoreService + remoteStoreNodeService ); final DiscoveryNode clusterManagerNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); @@ -519,7 +519,7 @@ public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService( new SetOnce<>(mock(RepositoriesService.class))::get, null ); @@ -529,7 +529,7 @@ public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception { allocationService, logger, rerouteService, - remoteStoreService + remoteStoreNodeService ); final DiscoveryNode clusterManagerNode = new DiscoveryNode( @@ -564,7 +564,7 @@ public void testUpdatesClusterStateWithMultiNodeCluster() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService( new SetOnce<>(mock(RepositoriesService.class))::get, null ); @@ -574,7 +574,7 @@ public void testUpdatesClusterStateWithMultiNodeCluster() throws Exception { allocationService, logger, rerouteService, - remoteStoreService + remoteStoreNodeService ); final DiscoveryNode clusterManagerNode = new DiscoveryNode( @@ -627,7 +627,7 @@ public void testUpdatesClusterStateWithSingleNodeClusterAndSameRepository() thro final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService( new SetOnce<>(mock(RepositoriesService.class))::get, null ); @@ -637,7 +637,7 @@ public void testUpdatesClusterStateWithSingleNodeClusterAndSameRepository() thro allocationService, logger, rerouteService, - remoteStoreService + remoteStoreNodeService ); final DiscoveryNode clusterManagerNode = new DiscoveryNode( @@ -672,7 +672,7 @@ public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throw final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + final RemoteStoreNodeService remoteStoreNodeService = new RemoteStoreNodeService( new SetOnce<>(mock(RepositoriesService.class))::get, null ); @@ -682,7 +682,7 @@ public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throw allocationService, logger, rerouteService, - remoteStoreService + remoteStoreNodeService ); final DiscoveryNode clusterManagerNode = new DiscoveryNode( diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index 014fc59e8e76d..7c0dca3803cb2 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -78,7 +78,7 @@ public class DiscoveryModuleTests extends OpenSearchTestCase { private ClusterSettings clusterSettings; private GatewayMetaState gatewayMetaState; - private RemoteStoreNodeService remoteStoreService; + private RemoteStoreNodeService remoteStoreNodeService; public interface DummyHostsProviderPlugin extends DiscoveryPlugin { Map> impl(); @@ -102,7 +102,7 @@ public void setupDummyServices() { clusterApplier = mock(ClusterApplier.class); clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); gatewayMetaState = mock(GatewayMetaState.class); - remoteStoreService = mock(RemoteStoreNodeService.class); + remoteStoreNodeService = mock(RemoteStoreNodeService.class); } @After @@ -127,7 +127,7 @@ private DiscoveryModule newModule(Settings settings, List plugi mock(RerouteService.class), null, new PersistedStateRegistry(), - remoteStoreService + remoteStoreNodeService ); } diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 11721a4b831bd..8de652138e83e 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -154,7 +154,7 @@ public class ClusterStateChanges { private final TransportClusterRerouteAction transportClusterRerouteAction; private final TransportCreateIndexAction transportCreateIndexAction; private final RepositoriesService repositoriesService; - private final RemoteStoreNodeService remoteStoreService; + private final RemoteStoreNodeService remoteStoreNodeService; private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor; private final JoinTaskExecutor joinTaskExecutor; @@ -376,10 +376,10 @@ public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version m threadPool ); - remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreNodeService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger); - joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, (s, p, r) -> {}, remoteStoreService); + joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, (s, p, r) -> {}, remoteStoreNodeService); } public ClusterState createIndex(ClusterState state, CreateIndexRequest request) { diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 289499e95e639..9cc7a83d3c563 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -1893,7 +1893,7 @@ private final class TestClusterNode { private final ClusterInfoService clusterInfoService; private Coordinator coordinator; - private RemoteStoreNodeService remoteStoreService; + private RemoteStoreNodeService remoteStoreNodeService; private Map actions = new HashMap<>(); @@ -1999,7 +1999,7 @@ public void onFailure(final Exception e) { emptyMap(), threadPool ); - remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreNodeService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService( settings, @@ -2517,7 +2517,7 @@ public void start(ClusterState initialState) { ElectionStrategy.DEFAULT_INSTANCE, () -> new StatusInfo(HEALTHY, "healthy-info"), persistedStateRegistry, - remoteStoreService + remoteStoreNodeService ); clusterManagerService.setClusterStatePublisher(coordinator); coordinator.start(); diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index d9a5a2e1f3fdd..d49d3d290b8a8 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -1039,7 +1039,7 @@ class ClusterNode { private DisruptableMockTransport mockTransport; private NodeHealthService nodeHealthService; private RepositoriesService repositoriesService; - private RemoteStoreNodeService remoteStoreService; + private RemoteStoreNodeService remoteStoreNodeService; List> extraJoinValidators = new ArrayList<>(); ClusterNode(int nodeIndex, boolean clusterManagerEligible, Settings nodeSettings, NodeHealthService nodeHealthService) { @@ -1142,7 +1142,7 @@ protected Optional getDisruptableMockTransport(Transpo Collections.emptyMap(), threadPool ); - remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreNodeService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); final Collection> onJoinValidators = Collections.singletonList( (dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)) ); @@ -1164,7 +1164,7 @@ protected Optional getDisruptableMockTransport(Transpo getElectionStrategy(), nodeHealthService, persistedStateRegistry, - remoteStoreService + remoteStoreNodeService ); clusterManagerService.setClusterStatePublisher(coordinator); final GatewayService gatewayService = new GatewayService( From b0a45231aff74b2cc2322bfe2e7f431922228296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 4 Sep 2023 01:42:05 +0530 Subject: [PATCH 22/34] Addressing comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../coordination/JoinTaskExecutor.java | 27 +++++-------------- .../metadata/MetadataCreateIndexService.java | 8 +++--- .../remotestore/RemoteStoreNodeAttribute.java | 6 +++++ .../remotestore/RemoteStoreNodeService.java | 13 ++++++++- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 65155002c2108..7541567d77071 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -46,7 +46,6 @@ import org.opensearch.cluster.routing.RerouteService; import org.opensearch.cluster.routing.allocation.AllocationService; import org.opensearch.common.Priority; -import org.opensearch.common.SetOnce; import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; @@ -176,7 +175,10 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); - SetOnce repositoriesMetadata = new SetOnce<>(); + RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( + joiningNodes.stream().filter(task -> task.node() != null).findAny(), + currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + ); assert nodesBuilder.isLocalNodeElectedClusterManager(); @@ -192,15 +194,6 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // noop } else if (currentNodes.nodeExistsWithSameRoles(node)) { logger.debug("received a join request for an existing node [{}]", node); - - if (repositoriesMetadata.get() == null) { - repositoriesMetadata.trySet( - remoteStoreNodeService.updateRepositoriesMetadata( - node, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE) - ) - ); - } } else { try { if (enforceMajorVersion) { @@ -220,14 +213,6 @@ public ClusterTasksResult execute(ClusterState currentState, List jo if (node.isClusterManagerNode()) { joiniedNodeNameIds.put(node.getName(), node.getId()); } - if (repositoriesMetadata.get() == null) { - repositoriesMetadata.trySet( - remoteStoreNodeService.updateRepositoriesMetadata( - node, - currentState.getMetadata().custom(RepositoriesMetadata.TYPE) - ) - ); - } } catch (IllegalArgumentException | IllegalStateException | NodeDecommissionedException e) { results.failure(joinTask, e); continue; @@ -291,8 +276,8 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } } - private Metadata updateMetadataWithRepositoriesMetadata(Metadata currentMetadata, SetOnce repositoriesMetadata) { - if (repositoriesMetadata == null || repositoriesMetadata.get() == null || repositoriesMetadata.get().repositories().isEmpty()) { + private Metadata updateMetadataWithRepositoriesMetadata(Metadata currentMetadata, RepositoriesMetadata repositoriesMetadata) { + if (repositoriesMetadata == null || repositoriesMetadata.repositories() == null || repositoriesMetadata.repositories().isEmpty()) { return currentMetadata; } else { return Metadata.builder(currentMetadata).putCustom(RepositoriesMetadata.TYPE, repositoriesMetadata.get()).build(); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 145eb15e4ebe9..c43353e9e64e0 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -135,6 +135,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent; /** * Service responsible for submitting create index requests @@ -937,9 +938,7 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, settingsBuilder.put(SETTING_REPLICATION_TYPE, INDEX_REPLICATION_TYPE_SETTING.get(requestSettings)); } else if (CLUSTER_REPLICATION_TYPE_SETTING.exists(clusterSettings)) { settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.get(clusterSettings)); - } else if (clusterSettings.getByPrefix( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX - ).isEmpty() == false) { + } else if (isRemoteStoreAttributePresent(clusterSettings)) { settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); } else { settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings)); @@ -952,8 +951,7 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, * @param clusterSettings cluster level settings */ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings clusterSettings) { - if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) - .isEmpty() == false) { + if (isRemoteStoreAttributePresent(clusterSettings)) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put( SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index e61c95edbe5fe..c7465d02e8efe 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -12,6 +12,7 @@ import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; +import org.opensearch.node.Node; import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.util.ArrayList; @@ -103,6 +104,11 @@ private RepositoriesMetadata buildRepositoriesMetadata() { return new RepositoriesMetadata(repositoryMetadataList); } + public static boolean isRemoteStoreAttributePresent(Settings settings) { + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) + .isEmpty() == false; + } + public RepositoriesMetadata getRepositoriesMetadata() { return this.repositoriesMetadata; } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 46aaca6e9cf59..e5e94d1872493 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.cluster.coordination.JoinTaskExecutor; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; @@ -22,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.function.Supplier; /** @@ -112,7 +114,16 @@ public List createAndVerifyRepositories(DiscoveryNode localNode) { * repository is already present in the cluster state and if it's different then the joining remote store backed * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ - public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode, RepositoriesMetadata existingRepositories) { + public RepositoriesMetadata updateRepositoriesMetadata( + Optional task, + RepositoriesMetadata existingRepositories + ) { + if (task.isEmpty()) { + return existingRepositories; + } + + DiscoveryNode joiningNode = task.get().node(); + if (joiningNode.isRemoteStoreNode()) { List updatedRepositoryMetadataList = new ArrayList<>(); List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode).getRepositoriesMetadata() From 401b6ecdead680bf8f0c4373b2684b9f9d7650da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 4 Sep 2023 03:44:48 +0530 Subject: [PATCH 23/34] Adding logic to clean up repository if create or verification fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../RemoteStoreBaseIntegTestCase.java | 16 +++-- .../multipart/RemoteStoreMultipartIT.java | 59 ++++++++++++++++++- .../remotestore/RemoteStoreNodeAttribute.java | 3 +- .../remotestore/RemoteStoreNodeService.java | 45 ++++++++------ .../repositories/RepositoriesService.java | 2 +- 5 files changed, 93 insertions(+), 32 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index d1d2cebc9bed2..495c52ed64763 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -263,12 +263,13 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel @After public void teardown() { clusterSettingsSuppliedByTest = false; - assertRemoteStoreRepositoryOnAllNodes(); + assertRemoteStoreRepositoryOnAllNodes(REPOSITORY_NAME); + assertRemoteStoreRepositoryOnAllNodes(REPOSITORY_2_NAME); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); } - private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + public RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { Map nodeAttributes = node.getAttributes(); String type = nodeAttributes.get(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); @@ -286,21 +287,18 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na return new RepositoryMetadata(name, type, settings.build()); } - private void assertRemoteStoreRepositoryOnAllNodes() { + public void assertRemoteStoreRepositoryOnAllNodes(String repositoryName) { RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) .state() .metadata() .custom(RepositoriesMetadata.TYPE); - RepositoryMetadata actualSegmentRepository = repositories.repository(REPOSITORY_NAME); - RepositoryMetadata actualTranslogRepository = repositories.repository(REPOSITORY_2_NAME); + RepositoryMetadata actualRepository = repositories.repository(repositoryName); for (String nodeName : internalCluster().getNodeNames()) { ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName); DiscoveryNode node = clusterService.localNode(); - RepositoryMetadata expectedSegmentRepository = buildRepositoryMetadata(node, REPOSITORY_NAME); - RepositoryMetadata expectedTranslogRepository = buildRepositoryMetadata(node, REPOSITORY_2_NAME); - assertTrue(actualSegmentRepository.equalsIgnoreGenerations(expectedSegmentRepository)); - assertTrue(actualTranslogRepository.equalsIgnoreGenerations(expectedTranslogRepository)); + RepositoryMetadata expectedRepository = buildRepositoryMetadata(node, repositoryName); + assertTrue(actualRepository.equalsIgnoreGenerations(expectedRepository)); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java index d7d80f90089d9..21f48ba99e651 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -9,6 +9,8 @@ package org.opensearch.remotestore.multipart; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.unit.ByteSizeUnit; @@ -16,29 +18,81 @@ import org.opensearch.remotestore.RemoteStoreIT; import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.junit.Before; import java.nio.file.Path; import java.util.Collection; +import java.util.Locale; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; public class RemoteStoreMultipartIT extends RemoteStoreIT { + Path repositoryLocation; + boolean compress; + boolean overrideBuildRepositoryMetadata; + @Override protected Collection> nodePlugins() { return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); } + @Before + public void setup() { + overrideBuildRepositoryMetadata = false; + repositoryLocation = randomRepoPath(); + compress = randomBoolean(); + } + + @Override + public RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + if (overrideBuildRepositoryMetadata) { + Map nodeAttributes = node.getAttributes(); + String type = nodeAttributes.get(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); + + String settingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + name + ); + Map settingsMap = node.getAttributes() + .keySet() + .stream() + .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> node.getAttributes().get(key))); + + Settings.Builder settings = Settings.builder(); + settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); + + if (name.equals(REPOSITORY_NAME)) { + settings.put("location", repositoryLocation) + .put("compress", compress) + .put("max_remote_upload_bytes_per_sec", "1kb") + .put("chunk_size", 100, ByteSizeUnit.BYTES); + return new RepositoryMetadata(name, MockFsRepositoryPlugin.TYPE, settings.build()); + } + + return new RepositoryMetadata(name, type, settings.build()); + } else { + return super.buildRepositoryMetadata(node, name); + } + } + public void testRateLimitedRemoteUploads() throws Exception { + overrideBuildRepositoryMetadata = true; internalCluster().startNode(); Client client = client(); logger.info("--> updating repository"); - Path repositoryLocation = randomRepoPath(); assertAcked( client.admin() .cluster() @@ -46,8 +100,9 @@ public void testRateLimitedRemoteUploads() throws Exception { .setType(MockFsRepositoryPlugin.TYPE) .setSettings( Settings.builder() + .put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true) .put("location", repositoryLocation) - .put("compress", randomBoolean()) + .put("compress", compress) .put("max_remote_upload_bytes_per_sec", "1kb") .put("chunk_size", 100, ByteSizeUnit.BYTES) ) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index c7465d02e8efe..7ed9d2984d9d7 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -105,8 +105,7 @@ private RepositoriesMetadata buildRepositoriesMetadata() { } public static boolean isRemoteStoreAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) - .isEmpty() == false; + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX).isEmpty() == false; } public RepositoriesMetadata getRepositoriesMetadata() { diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index e5e94d1872493..b75dbb5f01eec 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -82,28 +82,37 @@ public RemoteStoreNodeService(Supplier repositoriesService, * Creates a repository during a node startup and performs verification by invoking verify method against * mentioned repository. This verification will happen on a local node to validate if the node is able to connect * to the repository with appropriate permissions. + * If the creation or verification fails this will close all the repositories this method created and throw + * exception. */ public List createAndVerifyRepositories(DiscoveryNode localNode) { RemoteStoreNodeAttribute nodeAttribute = new RemoteStoreNodeAttribute(localNode); + RepositoriesService reposService = repositoriesService.get(); List repositories = new ArrayList<>(); for (RepositoryMetadata repositoryMetadata : nodeAttribute.getRepositoriesMetadata().repositories()) { String repositoryName = repositoryMetadata.name(); + Repository repository; + try { + RepositoriesService.validate(repositoryName); + + // Create Repository + repository = reposService.createRepository(repositoryMetadata); + logger.info( + "remote backed storage repository with name [{}] and type [{}] created", + repository.getMetadata().name(), + repository.getMetadata().type() + ); - // Create Repository - RepositoriesService.validate(repositoryName); - Repository repository = repositoriesService.get().createRepository(repositoryMetadata); - logger.info( - "remote backed storage repository with name {} and type {} created.", - repository.getMetadata().name(), - repository.getMetadata().type() - ); - - // Verify Repository - String verificationToken = repository.startVerification(); - repository.verify(verificationToken, localNode); - repository.endVerification(verificationToken); - logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); - + // Verify Repository + String verificationToken = repository.startVerification(); + repository.verify(verificationToken, localNode); + repository.endVerification(verificationToken); + logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); + } catch (Exception exception) { + logger.info(() -> new ParameterizedMessage("failure while creating and verifying [{}] repository", repositoryName)); + repositories.forEach(repo -> reposService.closeRepository(repo)); + throw exception; + } repositories.add(repository); } return repositories; @@ -115,14 +124,14 @@ public List createAndVerifyRepositories(DiscoveryNode localNode) { * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ public RepositoriesMetadata updateRepositoriesMetadata( - Optional task, + Optional joiningTask, RepositoriesMetadata existingRepositories ) { - if (task.isEmpty()) { + if (joiningTask.isEmpty()) { return existingRepositories; } - DiscoveryNode joiningNode = task.get().node(); + DiscoveryNode joiningNode = joiningTask.get().node(); if (joiningNode.isRemoteStoreNode()) { List updatedRepositoryMetadataList = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 256c4694dfd3e..e45ffb4d5e625 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -587,7 +587,7 @@ public void unregisterInternalRepository(String name) { } /** Closes the given repository. */ - private void closeRepository(Repository repository) { + public void closeRepository(Repository repository) { logger.debug("closing repository [{}][{}]", repository.getMetadata().type(), repository.getMetadata().name()); repository.close(); } From 8bed219b8ae4ed7abfd228169917a5ea37fcdc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 4 Sep 2023 13:15:49 +0530 Subject: [PATCH 24/34] Fixing nit issues to reduce overall code changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteIndexRecoveryIT.java | 2 +- .../remotestore/RemoteRestoreSnapshotIT.java | 10 +-- .../RemoteStoreBaseIntegTestCase.java | 61 +++++++++++++++++-- .../remotestore/RemoteStoreRestoreIT.java | 56 +++-------------- .../ReplicaToPrimaryPromotionIT.java | 2 +- .../opensearch/discovery/DiscoveryModule.java | 4 -- .../main/java/org/opensearch/node/Node.java | 7 +-- .../remotestore/RemoteStoreNodeService.java | 3 +- .../BlobStoreRepositoryRemoteIndexTests.java | 2 +- .../opensearch/test/InternalTestCluster.java | 4 +- 10 files changed, 79 insertions(+), 72 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java index e9539bb4fe049..d11c09928a08f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -42,7 +42,7 @@ public void setup() { protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, repositoryPath, REPOSITORY_NAME, repositoryPath)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, repositoryPath)) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 33c59c2850388..346e9d12410b7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -30,6 +30,7 @@ import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.nio.file.Path; @@ -48,17 +49,18 @@ public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase { private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; private Path remoteRepoPath; + @Before + public void setup() { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + } + @After public void teardown() { - remoteRepoPath = null; assertAcked(clusterAdmin().prepareDeleteRepository(BASE_REMOTE_REPO)); } @Override protected Settings nodeSettings(int nodeOrdinal) { - if (remoteRepoPath == null) { - remoteRepoPath = randomRepoPath().toAbsolutePath(); - } return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(FeatureFlags.REMOTE_STORE, "true") diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 495c52ed64763..7173fda89505c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -23,10 +23,13 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.Node; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; @@ -187,7 +190,7 @@ public static Settings remoteStoreClusterSettings( settingsBuilder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT); } - settingsBuilder.put(buildRemoteStoreNodeAttributes(segmentRepoName, segmentRepoPath, translogRepoName, translogRepoPath)); + settingsBuilder.put(buildRemoteStoreNodeAttributes(segmentRepoName, segmentRepoPath, translogRepoName, translogRepoPath, false)); return settingsBuilder.build(); } @@ -196,7 +199,8 @@ public static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, Path segmentRepoPath, String translogRepoName, - Path translogRepoPath + Path translogRepoPath, + boolean withRateLimiterAttributes ) { String segmentRepoTypeAttributeKey = String.format( Locale.getDefault(), @@ -219,14 +223,21 @@ public static Settings buildRemoteStoreNodeAttributes( translogRepoName ); - return Settings.builder() + Settings.Builder settings = Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) .put(translogRepoTypeAttributeKey, FsRepository.TYPE) - .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) - .build(); + .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath); + + if (withRateLimiterAttributes) { + settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) + .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "2kb") + .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES); + } + + return settings.build(); } private Settings defaultIndexSettings() { @@ -302,6 +313,46 @@ public void assertRemoteStoreRepositoryOnAllNodes(String repositoryName) { } } + public Settings buildClusterSettingsWith() { + String segmentRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + String segmentRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); + String translogRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_2_NAME + ); + String translogRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_2_NAME + ); + return Settings.builder() + .put( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REPOSITORY_NAME + ) + .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) + .put(segmentRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) + .put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) + .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "2kb") + .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES) + .put( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REPOSITORY_2_NAME + ) + .put(translogRepoTypeAttributeKey, FsRepository.TYPE) + .put(translogRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) + .build(); + } + public static int getFileCount(Path path) throws Exception { final AtomicInteger filesExisting = new AtomicInteger(0); Files.walkFileTree(path, new SimpleFileVisitor<>() { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 595a28761d100..b84b9e38c63a3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -15,12 +15,8 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.common.unit.ByteSizeUnit; -import org.opensearch.node.Node; -import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.RepositoriesService; -import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -29,13 +25,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; -import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThan; @@ -469,50 +462,17 @@ public void testRTSRestoreDataOnlyInTranslog() throws Exception { testRestoreFlow(0, true, randomIntBetween(1, 5)); } - private Settings buildClusterSettings() { - String segmentRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - REPOSITORY_NAME - ); - String segmentRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - REPOSITORY_NAME - ); - String translogRepoTypeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - REPOSITORY_2_NAME - ); - String translogRepoSettingsAttributeKeyPrefix = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - REPOSITORY_2_NAME - ); - return Settings.builder() - .put( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, - REPOSITORY_NAME - ) - .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) - .put(segmentRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) - .put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) - .put(segmentRepoSettingsAttributeKeyPrefix + "max_remote_download_bytes_per_sec", "2kb") - .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES) - .put( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, - REPOSITORY_2_NAME - ) - .put(translogRepoTypeAttributeKey, FsRepository.TYPE) - .put(translogRepoSettingsAttributeKeyPrefix + "location", randomRepoPath()) - .build(); - } - public void testRateLimitedRemoteDownloads() throws Exception { clusterSettingsSuppliedByTest = true; int shardCount = randomIntBetween(1, 3); - prepareCluster(1, 3, INDEX_NAME, 0, shardCount, buildClusterSettings()); + prepareCluster( + 1, + 3, + INDEX_NAME, + 0, + shardCount, + buildRemoteStoreNodeAttributes(REPOSITORY_NAME, randomRepoPath(), REPOSITORY_2_NAME, randomRepoPath(), true) + ); Map indexStats = indexData(5, false, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index 4959a74b75ad4..8290cf5d61a39 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -99,7 +99,7 @@ public void testPromoteReplicaToPrimary() throws Exception { final DiscoveryNode randomNode = state.nodes().resolveNode(primaryShard.currentNodeId()); // stop the random data node, all remaining shards are promoted to primaries - internalCluster().stopCurrentClusterManagerNode(); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(randomNode.getName())); ensureYellowAndNoInitializingShards(indexName); state = client(internalCluster().getClusterManagerName()).admin().cluster().prepareState().get().getState(); diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index 4c60562e0b7dc..288371aa240a0 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -210,12 +210,8 @@ public DiscoveryModule( rerouteService, electionStrategy, nodeHealthService, -<<<<<<< HEAD persistedStateRegistry, - remoteStoreService -======= remoteStoreNodeService ->>>>>>> 5e3b618bf6b (Addressing Comments) ); } else { throw new IllegalArgumentException("Unknown discovery type [" + discoveryType + "]"); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 0a239243256ff..b9c5c3352adc9 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -270,7 +270,7 @@ import static org.opensearch.common.util.FeatureFlags.TELEMETRY; import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreAttributePresent; /** * A node represent a node within a cluster ({@code cluster.name}). The {@link #client()} can be used @@ -1754,10 +1754,7 @@ private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStore @Override public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { - if (Node.NODE_ATTRIBUTES.getAsMap(settings) - .keySet() - .stream() - .anyMatch(key -> key.startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX))) { + if (isRemoteStoreAttributePresent(settings)) { localNode.set( DiscoveryNode.createRemoteNodeLocal( settings, diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index b75dbb5f01eec..351a8b233125b 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -48,8 +48,7 @@ public class RemoteStoreNodeService { * @opensearch.internal */ public enum CompatibilityMode { - STRICT("strict"), - ALLOW_MIX("allow_mix"); + STRICT("strict"); public final String mode; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 39900e44856ee..0f24d60993f2f 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -87,7 +87,7 @@ protected Settings nodeSettings() { .build(); } - public static Settings buildRemoteStoreNodeAttributes(String repoName, Path repoPath) { + private Settings buildRemoteStoreNodeAttributes(String repoName, Path repoPath) { String repoTypeAttributeKey = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 95832dc9544ce..68299b168cc2f 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -92,6 +92,7 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLockObtainFailedException; +import org.opensearch.gateway.GatewayService; import org.opensearch.http.HttpServerTransport; import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressure; @@ -1320,7 +1321,8 @@ public synchronized void validateClusterFormed() { } }); states.forEach(cs -> { - if (cs.nodes().getNodes().values().stream().findFirst().get().isRemoteStoreNode()) { + if (cs.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) == false + && cs.nodes().getNodes().values().stream().findFirst().get().isRemoteStoreNode()) { RepositoriesMetadata repositoriesMetadata = cs.metadata().custom(RepositoriesMetadata.TYPE); assertTrue(repositoriesMetadata != null && !repositoriesMetadata.repositories().isEmpty()); } From eb0b0494e0b8cb3d1d4ba53bd345f4e502918f9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 4 Sep 2023 16:45:16 +0530 Subject: [PATCH 25/34] fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../opensearch/node/remotestore/RemoteStoreNodeService.java | 2 +- .../main/java/org/opensearch/test/InternalTestCluster.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 351a8b233125b..216f9a9bb2d59 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -108,7 +108,7 @@ public List createAndVerifyRepositories(DiscoveryNode localNode) { repository.endVerification(verificationToken); logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); } catch (Exception exception) { - logger.info(() -> new ParameterizedMessage("failure while creating and verifying [{}] repository", repositoryName)); + logger.error(() -> new ParameterizedMessage("failure while creating and verifying [{}] repository", repositoryName)); repositories.forEach(repo -> reposService.closeRepository(repo)); throw exception; } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 68299b168cc2f..95832dc9544ce 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -92,7 +92,6 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.env.ShardLockObtainFailedException; -import org.opensearch.gateway.GatewayService; import org.opensearch.http.HttpServerTransport; import org.opensearch.index.IndexService; import org.opensearch.index.IndexingPressure; @@ -1321,8 +1320,7 @@ public synchronized void validateClusterFormed() { } }); states.forEach(cs -> { - if (cs.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK) == false - && cs.nodes().getNodes().values().stream().findFirst().get().isRemoteStoreNode()) { + if (cs.nodes().getNodes().values().stream().findFirst().get().isRemoteStoreNode()) { RepositoriesMetadata repositoriesMetadata = cs.metadata().custom(RepositoriesMetadata.TYPE); assertTrue(repositoriesMetadata != null && !repositoriesMetadata.repositories().isEmpty()); } From c3717b0741eee139022661a24a010e7168d76dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 4 Sep 2023 17:57:12 +0530 Subject: [PATCH 26/34] Reverting the exception handling during repository creation and verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteStoreNodeService.java | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 216f9a9bb2d59..6392cd4926a46 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -91,27 +91,21 @@ public List createAndVerifyRepositories(DiscoveryNode localNode) { for (RepositoryMetadata repositoryMetadata : nodeAttribute.getRepositoriesMetadata().repositories()) { String repositoryName = repositoryMetadata.name(); Repository repository; - try { - RepositoriesService.validate(repositoryName); - - // Create Repository - repository = reposService.createRepository(repositoryMetadata); - logger.info( - "remote backed storage repository with name [{}] and type [{}] created", - repository.getMetadata().name(), - repository.getMetadata().type() - ); - - // Verify Repository - String verificationToken = repository.startVerification(); - repository.verify(verificationToken, localNode); - repository.endVerification(verificationToken); - logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); - } catch (Exception exception) { - logger.error(() -> new ParameterizedMessage("failure while creating and verifying [{}] repository", repositoryName)); - repositories.forEach(repo -> reposService.closeRepository(repo)); - throw exception; - } + RepositoriesService.validate(repositoryName); + + // Create Repository + repository = reposService.createRepository(repositoryMetadata); + logger.info( + "remote backed storage repository with name [{}] and type [{}] created", + repository.getMetadata().name(), + repository.getMetadata().type() + ); + + // Verify Repository + String verificationToken = repository.startVerification(); + repository.verify(verificationToken, localNode); + repository.endVerification(verificationToken); + logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); repositories.add(repository); } return repositories; From c4cb7806e6617622fe7f6a3fc98cff71e59087d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Mon, 4 Sep 2023 19:26:16 +0530 Subject: [PATCH 27/34] Adding cluster manager node creation to avoid flaky test as this had only two nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../opensearch/remotestore/ReplicaToPrimaryPromotionIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index 8290cf5d61a39..4e3f01b8f257f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -22,6 +22,7 @@ import org.opensearch.test.BackgroundIndexer; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.Before; import java.util.Locale; import java.util.concurrent.CountDownLatch; @@ -37,6 +38,11 @@ public class ReplicaToPrimaryPromotionIT extends RemoteStoreBaseIntegTestCase { private int shard_count = 5; + @Before + public void setup() { + internalCluster().startClusterManagerOnlyNode(); + } + @Override public Settings indexSettings() { return Settings.builder().put(super.indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shard_count).build(); From b06dab55a357ba2b2800409a8f8dcc78ee8b3824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Tue, 5 Sep 2023 01:40:39 +0530 Subject: [PATCH 28/34] Addressing comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../coordination/JoinTaskExecutor.java | 17 ++++--- .../remotestore/RemoteStoreNodeAttribute.java | 48 +++++++++++-------- .../remotestore/RemoteStoreNodeService.java | 13 +---- .../coordination/JoinTaskExecutorTests.java | 12 +++-- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 7541567d77071..39385c9a38aa4 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -175,10 +175,11 @@ public ClusterTasksResult execute(ClusterState currentState, List jo } DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); - RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( - joiningNodes.stream().filter(task -> task.node() != null).findAny(), - currentState.getMetadata().custom(RepositoriesMetadata.TYPE) - ); + + // TODO: repositoriesMetadata is getting computed every time for individual set of joinTasks, We can optimize + // this to avoid computation if the repository metadata is already present. + RepositoriesMetadata repositoriesMetadata = null; + final RepositoriesMetadata existingRepositoriesMetadata = currentState.getMetadata().custom(RepositoriesMetadata.TYPE); assert nodesBuilder.isLocalNodeElectedClusterManager(); @@ -194,6 +195,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // noop } else if (currentNodes.nodeExistsWithSameRoles(node)) { logger.debug("received a join request for an existing node [{}]", node); + repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata(node, existingRepositoriesMetadata); } else { try { if (enforceMajorVersion) { @@ -213,6 +215,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo if (node.isClusterManagerNode()) { joiniedNodeNameIds.put(node.getName(), node.getId()); } + repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata(node, existingRepositoriesMetadata); } catch (IllegalArgumentException | IllegalStateException | NodeDecommissionedException e) { results.failure(joinTask, e); continue; @@ -488,11 +491,7 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { List existingNodes = new ArrayList<>(currentNodes.getNodes().values()); - // If there are no node in the cluster state we will No op the compatibility check as at this point we cannot - // determine if this is a remote store cluster or non-remote store cluster. - if (existingNodes.isEmpty()) { - return; - } + assert existingNodes.isEmpty() == false; // TODO: The below check is valid till we don't support migration, once we start supporting migration a remote // store node will be able to join a non remote store cluster and vice versa. #7986 diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 7ed9d2984d9d7..990692a4930b4 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -37,26 +38,24 @@ public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings."; private final RepositoriesMetadata repositoriesMetadata; - private final DiscoveryNode node; /** * Creates a new {@link RemoteStoreNodeAttribute} */ public RemoteStoreNodeAttribute(DiscoveryNode node) { - this.node = node; - this.repositoriesMetadata = buildRepositoriesMetadata(); + this.repositoriesMetadata = buildRepositoriesMetadata(node); } - private String validateAttributeNonNull(String attributeKey) { + private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) { String attributeValue = node.getAttributes().get(attributeKey); if (attributeValue == null || attributeValue.isEmpty()) { - throw new IllegalStateException("joining node [" + this.node + "] doesn't have the node attribute [" + attributeKey + "]"); + throw new IllegalStateException("joining node [" + node + "] doesn't have the node attribute [" + attributeKey + "]"); } return attributeValue; } - private Map validateSettingsAttributesNonNull(String repositoryName) { + private Map validateSettingsAttributesNonNull(DiscoveryNode node, String repositoryName) { String settingsAttributeKeyPrefix = String.format( Locale.getDefault(), REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, @@ -66,20 +65,23 @@ private Map validateSettingsAttributesNonNull(String repositoryN .keySet() .stream() .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) - .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(key))); + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(node, key))); if (settingsMap.isEmpty()) { throw new IllegalStateException( - "joining node [" + this.node + "] doesn't have settings attribute for [" + repositoryName + "] repository" + "joining node [" + node + "] doesn't have settings attribute for [" + repositoryName + "] repository" ); } return settingsMap; } - private RepositoryMetadata buildRepositoryMetadata(String name) { - String type = validateAttributeNonNull(String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name)); - Map settingsMap = validateSettingsAttributesNonNull(name); + private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name) { + String type = validateAttributeNonNull( + node, + String.format(Locale.getDefault(), REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, name) + ); + Map settingsMap = validateSettingsAttributesNonNull(node, name); Settings.Builder settings = Settings.builder(); settingsMap.forEach(settings::put); @@ -90,15 +92,15 @@ private RepositoryMetadata buildRepositoryMetadata(String name) { return new RepositoryMetadata(name, type, settings.build()); } - private RepositoriesMetadata buildRepositoriesMetadata() { + private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { List repositoryMetadataList = new ArrayList<>(); Set repositoryNames = new HashSet<>(); - repositoryNames.add(validateAttributeNonNull(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); - repositoryNames.add(validateAttributeNonNull(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); for (String repositoryName : repositoryNames) { - repositoryMetadataList.add(buildRepositoryMetadata(repositoryName)); + repositoryMetadataList.add(buildRepositoryMetadata(node, repositoryName)); } return new RepositoriesMetadata(repositoryMetadataList); @@ -114,9 +116,18 @@ public RepositoriesMetadata getRepositoriesMetadata() { @Override public int hashCode() { - // We will hash the id and repositories metadata as its highly unlikely that two nodes will have same id and - // repositories metadata but are actually different. - return Objects.hash(node.getEphemeralId(), repositoriesMetadata); + // The hashCode is generated by computing the has of all the repositoryMetadata present in + // repositoriesMetadata without generation. Below is the modified list hashCode generation logic. + + int hashCode = 1; + Iterator iterator = this.repositoriesMetadata.repositories().iterator(); + while (iterator.hasNext()) { + RepositoryMetadata repositoryMetadata = (RepositoryMetadata) iterator.next(); + hashCode = 31 * hashCode + (repositoryMetadata == null + ? 0 + : Objects.hash(repositoryMetadata.name(), repositoryMetadata.type(), repositoryMetadata.settings())); + } + return hashCode; } @Override @@ -132,7 +143,6 @@ public boolean equals(Object o) { @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append('{').append(this.node).append('}'); sb.append('{').append(this.repositoriesMetadata).append('}'); return super.toString(); } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 6392cd4926a46..9e107e196b262 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.cluster.coordination.JoinTaskExecutor; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; @@ -23,7 +22,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Optional; import java.util.function.Supplier; /** @@ -116,16 +114,7 @@ public List createAndVerifyRepositories(DiscoveryNode localNode) { * repository is already present in the cluster state and if it's different then the joining remote store backed * node repository metadata an exception will be thrown and the node will not be allowed to join the cluster. */ - public RepositoriesMetadata updateRepositoriesMetadata( - Optional joiningTask, - RepositoriesMetadata existingRepositories - ) { - if (joiningTask.isEmpty()) { - return existingRepositories; - } - - DiscoveryNode joiningNode = joiningTask.get().node(); - + public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode, RepositoriesMetadata existingRepositories) { if (joiningNode.isRemoteStoreNode()) { List updatedRepositoryMetadataList = new ArrayList<>(); List newRepositoryMetadataList = new RemoteStoreNodeAttribute(joiningNode).getRepositoriesMetadata() diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 38cd0a1cf6843..e0844fd521d3f 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -363,16 +363,20 @@ public void testJoinClusterWithDecommissionFailed() { } public void testJoinClusterWithNonRemoteStoreNodeJoining() { - ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).build(); - DiscoveryNode joiningNode = newDiscoveryNode(Collections.emptyMap()); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(joiningNode).build()) + .build(); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } public void testJoinClusterWithRemoteStoreNodeJoining() { - ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT).build(); - DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO)); + ClusterState currentState = ClusterState.builder(ClusterName.DEFAULT) + .nodes(DiscoveryNodes.builder().add(joiningNode).build()) + .build(); + JoinTaskExecutor.ensureNodesCompatibility(joiningNode, currentState.getNodes(), currentState.metadata()); } From e243e18788a791135fc2f9dbea65976174ca2167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Tue, 5 Sep 2023 12:42:05 +0530 Subject: [PATCH 29/34] Using local node to compute repository metadata till we only support Strict Compatibility Mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../coordination/JoinTaskExecutor.java | 20 +++++++++---------- .../remotestore/RemoteStoreNodeAttribute.java | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 39385c9a38aa4..938884100b1b0 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -176,10 +176,14 @@ public ClusterTasksResult execute(ClusterState currentState, List jo DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); - // TODO: repositoriesMetadata is getting computed every time for individual set of joinTasks, We can optimize - // this to avoid computation if the repository metadata is already present. - RepositoriesMetadata repositoriesMetadata = null; - final RepositoriesMetadata existingRepositoriesMetadata = currentState.getMetadata().custom(RepositoriesMetadata.TYPE); + // TODO: We are using the local node to build the repository metadata, this will need to be updated once + // we start supporting mixed compatibility mode. + // An optimization can be done as this will get invoked for every set of node join task which we can optimize + // to not compute if cluster state already has repository information. + RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( + currentNodes.getLocalNode(), + currentState.getMetadata().custom(RepositoriesMetadata.TYPE) + ); assert nodesBuilder.isLocalNodeElectedClusterManager(); @@ -195,7 +199,6 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // noop } else if (currentNodes.nodeExistsWithSameRoles(node)) { logger.debug("received a join request for an existing node [{}]", node); - repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata(node, existingRepositoriesMetadata); } else { try { if (enforceMajorVersion) { @@ -215,7 +218,6 @@ public ClusterTasksResult execute(ClusterState currentState, List jo if (node.isClusterManagerNode()) { joiniedNodeNameIds.put(node.getName(), node.getId()); } - repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata(node, existingRepositoriesMetadata); } catch (IllegalArgumentException | IllegalStateException | NodeDecommissionedException e) { results.failure(joinTask, e); continue; @@ -489,15 +491,11 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) * needs to be modified. */ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { - List existingNodes = new ArrayList<>(currentNodes.getNodes().values()); - - assert existingNodes.isEmpty() == false; - // TODO: The below check is valid till we don't support migration, once we start supporting migration a remote // store node will be able to join a non remote store cluster and vice versa. #7986 CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); if (STRICT.equals(remoteStoreCompatibilityMode)) { - DiscoveryNode existingNode = existingNodes.get(0); + DiscoveryNode existingNode = currentNodes.getLocalNode(); if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 990692a4930b4..b514089966484 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -116,7 +116,7 @@ public RepositoriesMetadata getRepositoriesMetadata() { @Override public int hashCode() { - // The hashCode is generated by computing the has of all the repositoryMetadata present in + // The hashCode is generated by computing the hash of all the repositoryMetadata present in // repositoriesMetadata without generation. Below is the modified list hashCode generation logic. int hashCode = 1; From 46ffa47d320a1137010e7a59f0318dc279815dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Tue, 5 Sep 2023 13:47:03 +0530 Subject: [PATCH 30/34] Using existing nodes instead of local node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../cluster/coordination/JoinTaskExecutor.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 938884100b1b0..c297b24843416 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -176,12 +176,12 @@ public ClusterTasksResult execute(ClusterState currentState, List jo DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); - // TODO: We are using the local node to build the repository metadata, this will need to be updated once - // we start supporting mixed compatibility mode. - // An optimization can be done as this will get invoked for every set of node join task which we can optimize - // to not compute if cluster state already has repository information. + // TODO: We are using one of the existing node to build the repository metadata, this will need to be updated + // once we start supporting mixed compatibility mode. An optimization can be done as this will get invoked + // for every set of node join task which we can optimize to not compute if cluster state already has + // repository information. RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( - currentNodes.getLocalNode(), + (currentNodes.getNodes().values()).stream().collect(Collectors.toList()).get(0), currentState.getMetadata().custom(RepositoriesMetadata.TYPE) ); @@ -491,11 +491,15 @@ public static void ensureNodeCommissioned(DiscoveryNode node, Metadata metadata) * needs to be modified. */ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNode, DiscoveryNodes currentNodes, Metadata metadata) { + List existingNodes = new ArrayList<>(currentNodes.getNodes().values()); + + assert existingNodes.isEmpty() == false; + // TODO: The below check is valid till we don't support migration, once we start supporting migration a remote // store node will be able to join a non remote store cluster and vice versa. #7986 CompatibilityMode remoteStoreCompatibilityMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()); if (STRICT.equals(remoteStoreCompatibilityMode)) { - DiscoveryNode existingNode = currentNodes.getLocalNode(); + DiscoveryNode existingNode = existingNodes.get(0); if (joiningNode.isRemoteStoreNode()) { if (existingNode.isRemoteStoreNode()) { RemoteStoreNodeAttribute joiningRemoteStoreNodeAttribute = new RemoteStoreNodeAttribute(joiningNode); From 41e78a76c3d1703299fd2f9307f4637e6780f104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Tue, 5 Sep 2023 16:09:30 +0530 Subject: [PATCH 31/34] Using stream find first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../org/opensearch/cluster/coordination/JoinTaskExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index c297b24843416..15eaf9c8bcc1e 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -181,7 +181,7 @@ public ClusterTasksResult execute(ClusterState currentState, List jo // for every set of node join task which we can optimize to not compute if cluster state already has // repository information. RepositoriesMetadata repositoriesMetadata = remoteStoreNodeService.updateRepositoriesMetadata( - (currentNodes.getNodes().values()).stream().collect(Collectors.toList()).get(0), + (currentNodes.getNodes().values()).stream().findFirst().get(), currentState.getMetadata().custom(RepositoriesMetadata.TYPE) ); From c43a1128fd41cb1fc817a02ded404be2692f89c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Tue, 5 Sep 2023 23:31:50 +0530 Subject: [PATCH 32/34] Changes to update RepositoriesService repositories with system repositories during node bootstrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../remotestore/RemoteStoreNodeService.java | 11 ++++-- .../repositories/RepositoriesService.java | 37 ++++++++++++++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java index 9e107e196b262..26c078353d12a 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java @@ -20,8 +20,10 @@ import org.opensearch.threadpool.ThreadPool; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.function.Supplier; /** @@ -82,10 +84,10 @@ public RemoteStoreNodeService(Supplier repositoriesService, * If the creation or verification fails this will close all the repositories this method created and throw * exception. */ - public List createAndVerifyRepositories(DiscoveryNode localNode) { + public void createAndVerifyRepositories(DiscoveryNode localNode) { RemoteStoreNodeAttribute nodeAttribute = new RemoteStoreNodeAttribute(localNode); RepositoriesService reposService = repositoriesService.get(); - List repositories = new ArrayList<>(); + Map repositories = new HashMap<>(); for (RepositoryMetadata repositoryMetadata : nodeAttribute.getRepositoriesMetadata().repositories()) { String repositoryName = repositoryMetadata.name(); Repository repository; @@ -104,9 +106,10 @@ public List createAndVerifyRepositories(DiscoveryNode localNode) { repository.verify(verificationToken, localNode); repository.endVerification(verificationToken); logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); - repositories.add(repository); + repositories.put(repositoryName, repository); } - return repositories; + // Updating the repositories map in RepositoriesService + reposService.updateRepositoriesMap(repositories); } /** diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index e45ffb4d5e625..9a68bba0ba289 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -420,7 +420,13 @@ public void applyClusterState(ClusterChangedEvent event) { // Check if repositories got changed if ((oldMetadata == null && newMetadata == null) || (oldMetadata != null && oldMetadata.equalsIgnoreGenerations(newMetadata))) { for (Repository repo : repositories.values()) { - repo.updateState(state); + // Update State should only be invoked for repository which are already in cluster state. This + // check needs to be added as system repositories can be populated before cluster state has the + // repository metadata. + RepositoriesMetadata stateRepositoriesMetadata = state.metadata().custom(RepositoriesMetadata.TYPE); + if (stateRepositoriesMetadata != null && stateRepositoriesMetadata.repository(repo.getMetadata().name()) != null) { + repo.updateState(state); + } } return; } @@ -468,7 +474,22 @@ public void applyClusterState(ClusterChangedEvent event) { } } else { try { - repository = createRepository(repositoryMetadata, typesRegistry); + // System repositories are already created and verified and hence during cluster state + // update we should avoid creating it again. Once the cluster state is update with the + // repository metadata the repository metadata update will land in the above if block. + if (repositories.containsKey(repositoryMetadata.name()) == false) { + repository = createRepository(repositoryMetadata, typesRegistry); + } else { + // Validate the repository metadata which was created during bootstrap is same as the + // one present in incoming cluster state. + repository = repositories.get(repositoryMetadata.name()); + if (repositoryMetadata.equalsIgnoreGenerations(repository.getMetadata()) == false) { + throw new RepositoryException( + repositoryMetadata.name(), + "repository was already " + "registered with different metadata during bootstrap than cluster state" + ); + } + } } catch (RepositoryException ex) { logger.warn(() -> new ParameterizedMessage("failed to create repository [{}]", repositoryMetadata.name()), ex); } @@ -731,6 +752,18 @@ private static boolean isRepositoryInUse(ClusterState clusterState, String repos return false; } + /** + * This method will be used to update the repositories map. + * @param repos + */ + public void updateRepositoriesMap(Map repos) { + if (repositories.isEmpty()) { + repositories = repos; + } else { + throw new IllegalArgumentException("can't overwrite as repositories are already present"); + } + } + @Override protected void doStart() { From 899ae5dfe7d6f2bc23d49e3ac7e4f44756d57fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Wed, 6 Sep 2023 00:03:47 +0530 Subject: [PATCH 33/34] Fixing precommit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../java/org/opensearch/repositories/RepositoriesService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java index 9a68bba0ba289..ab343aabd3a82 100644 --- a/server/src/main/java/org/opensearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/opensearch/repositories/RepositoriesService.java @@ -754,7 +754,6 @@ private static boolean isRepositoryInUse(ClusterState clusterState, String repos /** * This method will be used to update the repositories map. - * @param repos */ public void updateRepositoriesMap(Map repos) { if (repositories.isEmpty()) { From 8c54a861cb11eef1def0ff9b74a79741532950fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Wed, 6 Sep 2023 11:54:58 +0530 Subject: [PATCH 34/34] Fixing flaky test #9776 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- .../stream/read/listener/ReadContextListenerTests.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java index f785b5f1191b4..936e2170bed6c 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java @@ -70,7 +70,7 @@ public void testReadContextListener() throws InterruptedException, IOException { assertEquals(NUMBER_OF_PARTS * PART_SIZE, Files.size(fileLocation)); } - public void testReadContextListenerFailure() throws InterruptedException { + public void testReadContextListenerFailure() throws Exception { Path fileLocation = path.resolve(UUID.randomUUID().toString()); List blobPartStreams = initializeBlobPartStreams(); CountDownLatch countDownLatch = new CountDownLatch(1); @@ -99,8 +99,7 @@ public int available() { readContextListener.onResponse(readContext); countDownLatch.await(); - - assertFalse(Files.exists(fileLocation)); + assertBusy(() -> { assertFalse(Files.exists(fileLocation)); }); } public void testReadContextListenerException() {