Add more ITs and handle some failure cases

Signed-off-by: Gaurav Bafna <[email protected]>
gbbafna · Sep 26, 2024 · 0382146 · 0382146
1 parent 3dc55e4
commit 0382146
Show file tree

Hide file tree

Showing 5 changed files with 588 additions and 61 deletions.
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
@@ -46,14 +46,11 @@
 import org.opensearch.repositories.RepositoryData;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.repositories.fs.FsRepository;
-import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.opensearch.snapshots.SnapshotInfo;
 import org.opensearch.snapshots.SnapshotRestoreException;
 import org.opensearch.snapshots.SnapshotState;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
-import org.junit.After;
-import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -83,48 +80,7 @@
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase {
-    private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
-    private Path remoteRepoPath;
-
-    @Before
-    public void setup() {
-        remoteRepoPath = randomRepoPath().toAbsolutePath();
-    }
-
-    @After
-    public void teardown() {
-        clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
-            .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath))
-            .build();
-    }
-
-    private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
-        Settings.Builder settingsBuilder = Settings.builder()
-            .put(super.indexSettings())
-            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
-            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
-            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
-        return settingsBuilder;
-    }
-
-    private void indexDocuments(Client client, String indexName, int numOfDocs) {
-        indexDocuments(client, indexName, 0, numOfDocs);
-    }
-
-    private void indexDocuments(Client client, String indexName, int fromId, int toId) {
-        for (int i = fromId; i < toId; i++) {
-            String id = Integer.toString(i);
-            client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
-        }
-        client.admin().indices().prepareFlush(indexName).get();
-    }
+public class RemoteRestoreSnapshotIT extends RemoteSnapshotIT {
 
     private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) {
         for (int i = 0; i < numOfDocs; i++) {
@@ -1388,11 +1344,4 @@ public void testConcurrentV1SnapshotAndV2RepoSettingUpdate() throws Exception {
         createV1SnapshotThread.join();
     }
 
-    private Settings pinnedTimestampSettings() {
-        Settings settings = Settings.builder()
-            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
-            .build();
-        return settings;
-    }
-
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java
@@ -0,0 +1,89 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotestore;
+
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
+import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.RepositoryMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.indices.RemoteStoreSettings;
+import org.opensearch.repositories.fs.ReloadableFsRepository;
+import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.nio.file.Path;
+import java.util.concurrent.ExecutionException;
+
+import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
+
+public abstract class RemoteSnapshotIT extends AbstractSnapshotIntegTestCase {
+    protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
+    protected Path remoteRepoPath;
+
+    @Before
+    public void setup() {
+        remoteRepoPath = randomRepoPath().toAbsolutePath();
+    }
+
+    @After
+    public void teardown() {
+        clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath))
+            .build();
+    }
+
+    protected Settings pinnedTimestampSettings() {
+        Settings settings = Settings.builder()
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
+            .build();
+        return settings;
+    }
+
+    protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
+        Settings.Builder settingsBuilder = Settings.builder()
+            .put(super.indexSettings())
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
+        return settingsBuilder;
+    }
+
+    protected void indexDocuments(Client client, String indexName, int numOfDocs) {
+        indexDocuments(client, indexName, 0, numOfDocs);
+    }
+
+    void indexDocuments(Client client, String indexName, int fromId, int toId) {
+        for (int i = fromId; i < toId; i++) {
+            String id = Integer.toString(i);
+            client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
+        }
+        client.admin().indices().prepareFlush(indexName).get();
+    }
+
+    protected void setFailRate(String repoName, int value) throws ExecutionException, InterruptedException {
+        GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { repoName });
+        GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get();
+        RepositoryMetadata rmd = res.repositories().get(0);
+        Settings.Builder settings = Settings.builder()
+            .put("location", rmd.settings().get("location"))
+            .put(REPOSITORIES_FAILRATE_SETTING.getKey(), value);
+        createRepository(repoName, ReloadableFsRepository.TYPE, settings);
+    }
+
+}