[Draft] Master recommission

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java

@@ -105,23 +105,25 @@ public OpenSearchCluster(
 
         // Always add the first node
         addNode(clusterName + "-0");
+        setNumberOfNodes(3);
         // configure the cluster name eagerly so all nodes know about it
         this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName)));
 
         addWaitForClusterHealth();
     }
 
     public void setNumberOfNodes(int numberOfNodes) {
+        numberOfNodes = 3;
         checkFrozen();
 
         if (numberOfNodes < 1) {
             throw new IllegalArgumentException("Number of nodes should be >= 1 but was " + numberOfNodes + " for " + this);
         }
 
         if (numberOfNodes <= nodes.size()) {
-            throw new IllegalArgumentException(
-                "Cannot shrink " + this + " to have " + numberOfNodes + " nodes as it already has " + getNumberOfNodes()
-            );
+//            throw new IllegalArgumentException(
+//                "Cannot shrink " + this + " to have " + numberOfNodes + " nodes as it already has " + getNumberOfNodes()
+//            );
         }
 
         for (int i = nodes.size(); i < numberOfNodes; i++) {

server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java

@@ -36,6 +36,12 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.action.ActionListener;
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsResponse;
+import org.opensearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
+import org.opensearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsResponse;
+import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
+import org.opensearch.action.admin.cluster.configuration.TransportClearVotingConfigExclusionsAction;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.ActiveShardCount;
 import org.opensearch.action.support.IndicesOptions;
@@ -49,6 +55,7 @@
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException;
+import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.routing.allocation.AllocationService;
 import org.opensearch.cluster.service.ClusterService;
@@ -57,13 +64,16 @@
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.CollectionUtils;
+import org.opensearch.discovery.Discovery;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.node.NodeClosedException;
+import org.opensearch.node.NodeService;
 import org.opensearch.tasks.Task;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -78,14 +88,23 @@ public class TransportClusterHealthAction extends TransportClusterManagerNodeRea
 
     private final AllocationService allocationService;
 
+    private final Discovery discovery;
+
+    private final TransportAddVotingConfigExclusionsAction exclusionsAction;
+
+    private final TransportClearVotingConfigExclusionsAction clearAction;
+
     @Inject
     public TransportClusterHealthAction(
         TransportService transportService,
         ClusterService clusterService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        AllocationService allocationService
+        AllocationService allocationService,
+        NodeService nodeService,
+        TransportAddVotingConfigExclusionsAction exclusionsAction,
+        TransportClearVotingConfigExclusionsAction clearAction
     ) {
         super(
             ClusterHealthAction.NAME,
@@ -98,6 +117,9 @@ public TransportClusterHealthAction(
             indexNameExpressionResolver
         );
         this.allocationService = allocationService;
+        this.discovery = nodeService.discovery;
+        this.exclusionsAction = exclusionsAction;
+        this.clearAction = clearAction;
     }
 
     @Override
@@ -131,6 +153,46 @@ protected void masterOperation(
         final ClusterState unusedState,
         final ActionListener<ClusterHealthResponse> listener
     ) {
+        //find the node .
+        String masterId = clusterService.state().getNodes().getMasterNodeId();
+        DiscoveryNode mast = clusterService.state().getNodes().get(masterId);
+        Map<String, String> attr =  mast.getAttributes();
+        logger.info("Master name is " + mast.getName() + "attributes" + attr);
+        // ToDo : Match zone values from attributes
+        if ( mast.getName() != "runTask-2" ) {
+            logger.info("changing  master ");
+            ActionListener<ClearVotingConfigExclusionsResponse> listener3 =  new ActionListener<>() {
+                @Override
+                public void onResponse(ClearVotingConfigExclusionsResponse clearVotingConfigExclusionsResponse) {
+                    logger.info("remoed   exclude di as well ");
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e);
+                }
+            };
+
+            ActionListener<AddVotingConfigExclusionsResponse> listener2 = new ActionListener<>() {
+
+                @Override
+                public void onResponse(AddVotingConfigExclusionsResponse addVotingConfigExclusionsResponse) {
+                    logger.info("trying to changes master ");
+                    ClearVotingConfigExclusionsRequest req = new ClearVotingConfigExclusionsRequest();
+                    req.setWaitForRemoval(false);
+                    clearAction.execute(req, listener3);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e);
+                }
+            };
+
+            exclusionsAction.execute(new AddVotingConfigExclusionsRequest(mast.getName()), listener2  );
+            logger.info("changed  master ");
+        }
+
 
         final int waitCount = getWaitCount(request);
 

server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java

@@ -71,8 +71,8 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.transport.TransportAddress;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.common.util.concurrent.ListenableFuture;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.discovery.Discovery;
@@ -512,7 +512,7 @@ private void startElection() {
         }
     }
 
-    private void abdicateTo(DiscoveryNode newClusterManager) {
+    public void abdicateTo(DiscoveryNode newClusterManager) {
         assert Thread.holdsLock(mutex);
         assert mode == Mode.LEADER : "expected to be leader on abdication but was " + mode;
         assert newClusterManager.isMasterNode() : "should only abdicate to cluster-manager-eligible node but was " + newClusterManager;
@@ -528,7 +528,6 @@ private void abdicateTo(DiscoveryNode newClusterManager) {
         // explicitly move node to candidate state so that the next cluster state update task yields an onNoLongerMaster event
         becomeCandidate("after abdicating to " + newClusterManager);
     }
-
     private static boolean localNodeMayWinElection(ClusterState lastAcceptedState) {
         final DiscoveryNode localNode = lastAcceptedState.nodes().getLocalNode();
         assert localNode != null;

server/src/main/java/org/opensearch/discovery/DiscoveryModule.java

@@ -113,7 +113,7 @@ public class DiscoveryModule {
         Property.NodeScope
     );
 
-    private final Discovery discovery;
+    public final Discovery discovery;
 
     public DiscoveryModule(
         Settings settings,

server/src/main/java/org/opensearch/node/NodeService.java

@@ -82,7 +82,7 @@ public class NodeService implements Closeable {
     private final IndexingPressureService indexingPressureService;
     private final AggregationUsageService aggregationUsageService;
 
-    private final Discovery discovery;
+    public final Discovery discovery;
 
     NodeService(
         Settings settings,

server/src/test/java/org/opensearch/cluster/health/ClusterStateHealthTests.java

@@ -33,6 +33,10 @@
 
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -68,10 +72,6 @@
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -83,13 +83,13 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import static org.opensearch.test.ClusterServiceUtils.createClusterService;
-import static org.opensearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.opensearch.test.ClusterServiceUtils.createClusterService;
+import static org.opensearch.test.ClusterServiceUtils.setState;
 
 public class ClusterStateHealthTests extends OpenSearchTestCase {
     private final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(
@@ -177,7 +177,10 @@ public void testClusterHealthWaitsForClusterStateApplication() throws Interrupte
             threadPool,
             new ActionFilters(new HashSet<>()),
             indexNameExpressionResolver,
-            new AllocationService(null, new TestGatewayAllocator(), null, null, null)
+            new AllocationService(null, new TestGatewayAllocator(), null, null, null),
+            null,
+            null,
+            null
         );
         PlainActionFuture<ClusterHealthResponse> listener = new PlainActionFuture<>();
         action.execute(new ClusterHealthRequest().waitForGreenStatus(), listener);