From 8951a314ee8ac9a11bc14468377fef2b61c9116a Mon Sep 17 00:00:00 2001 From: Xiaxuan Gao Date: Wed, 1 Nov 2023 17:11:43 -0700 Subject: [PATCH] Rewrite the filtering logic by adding evacuating instances to toBeStoppedInstances list --- .../helix/util/InstanceValidationUtil.java | 25 +--- .../util/TestInstanceValidationUtil.java | 80 ----------- .../MaintenanceManagementService.java | 110 +++++++++++++-- .../MaintenanceManagementServiceBuilder.java | 127 ------------------ .../StoppableInstancesSelector.java | 38 +++++- .../resources/helix/InstancesAccessor.java | 44 +++--- .../TestMaintenanceManagementService.java | 21 +-- .../rest/server/TestInstancesAccessor.java | 95 ++++++++----- .../TestInstanceValidationUtilInRest.java | 41 ++---- 9 files changed, 227 insertions(+), 354 deletions(-) delete mode 100644 helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementServiceBuilder.java diff --git a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java index 6c542ab7bd..aef1bcac75 100644 --- a/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java +++ b/helix-core/src/main/java/org/apache/helix/util/InstanceValidationUtil.java @@ -240,23 +240,6 @@ public static boolean hasErrorPartitions(HelixDataAccessor dataAccessor, String return false; } - /** - * Checks if the specified instance is marked for an ongoing instance operation. Currently, - * this method only checks for evacuation. - * - * @param dataAccessor The accessor for retrieving Helix data properties. - * @param instanceName An instance to be evaluated. - * @return - */ - public static boolean isOperationSetForInstance(HelixDataAccessor dataAccessor, - String instanceName) { - PropertyKey.Builder propertyKeyBuilder = dataAccessor.keyBuilder(); - InstanceConfig instanceConfig = - dataAccessor.getProperty(propertyKeyBuilder.instanceConfig(instanceName)); - return InstanceConstants.InstanceOperation.EVACUATE.name() - .equals(instanceConfig.getInstanceOperation()); - } - /** * Get the problematic partitions on the to-be-stop instance * Requirement: @@ -319,11 +302,6 @@ public static Map> perPartitionHealthCheck(List new ArrayList<>()) @@ -477,8 +455,7 @@ public static boolean siblingNodesActiveReplicaCheck(HelixDataAccessor dataAcces String siblingInstanceName = entry.getKey(); if (!siblingInstanceName.equals(instanceName) && (toBeStoppedInstances == null || !toBeStoppedInstances.contains(siblingInstanceName)) - && !unhealthyStates.contains(entry.getValue()) && !isOperationSetForInstance( - dataAccessor, siblingInstanceName)) { + && !unhealthyStates.contains(entry.getValue())) { numHealthySiblings++; } } diff --git a/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java b/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java index 1ffba3c3d4..79b0fdce81 100644 --- a/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java +++ b/helix-core/src/test/java/org/apache/helix/util/TestInstanceValidationUtil.java @@ -33,7 +33,6 @@ import org.apache.helix.HelixException; import org.apache.helix.PropertyKey; import org.apache.helix.PropertyType; -import org.apache.helix.constants.InstanceConstants; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.CurrentState; import org.apache.helix.model.ExternalView; @@ -46,7 +45,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.argThat; @@ -399,9 +397,6 @@ public void TestSiblingNodesActiveReplicaCheck_success() { doReturn(stateModelDefinition).when(mock.dataAccessor) .getProperty(argThat(new PropertyKeyArgument(PropertyType.STATEMODELDEFS))); - // set up default instances config - setDefaultInstanceConfigs(mock); - boolean result = InstanceValidationUtil.siblingNodesActiveReplicaCheck(mock.dataAccessor, TEST_INSTANCE); @@ -435,9 +430,6 @@ public void TestSiblingNodesActiveReplicaCheckSuccessWithToBeStoppedInstances() doReturn(stateModelDefinition).when(mock.dataAccessor) .getProperty(argThat(new PropertyKeyArgument(PropertyType.STATEMODELDEFS))); - // set default instance config - setDefaultInstanceConfigs(mock); - Set toBeStoppedInstances = new HashSet<>(); toBeStoppedInstances.add("instance3"); toBeStoppedInstances.add("invalidInstances"); // include an invalid instance. @@ -477,9 +469,6 @@ public void TestSiblingNodesActiveReplicaCheckFailsWithToBeStoppedInstances() { doReturn(stateModelDefinition).when(mock.dataAccessor) .getProperty(argThat(new PropertyKeyArgument(PropertyType.STATEMODELDEFS))); - // set default instance config - setDefaultInstanceConfigs(mock); - Set toBeStoppedInstances = new HashSet<>(); toBeStoppedInstances.add("instance1"); toBeStoppedInstances.add("instance2"); @@ -489,59 +478,6 @@ public void TestSiblingNodesActiveReplicaCheckFailsWithToBeStoppedInstances() { Assert.assertFalse(result); } - @Test - public void TestSiblingNodesActiveReplicaCheckFailsWithEvacuatingInstance() { - String resource = "resource"; - Mock mock = new Mock(); - doReturn(ImmutableList.of(resource)).when(mock.dataAccessor) - .getChildNames(argThat(new PropertyKeyArgument(PropertyType.IDEALSTATES))); - // set ideal state - IdealState idealState = mock(IdealState.class); - when(idealState.isEnabled()).thenReturn(true); - when(idealState.isValid()).thenReturn(true); - when(idealState.getStateModelDefRef()).thenReturn("MasterSlave"); - doReturn(idealState).when(mock.dataAccessor).getProperty(argThat(new PropertyKeyArgument(PropertyType.IDEALSTATES))); - - // set external view - ExternalView externalView = mock(ExternalView.class); - when(externalView.getMinActiveReplicas()).thenReturn(2); - when(externalView.getStateModelDefRef()).thenReturn("MasterSlave"); - when(externalView.getPartitionSet()).thenReturn(ImmutableSet.of("db0")); - when(externalView.getStateMap("db0")).thenReturn(ImmutableMap.of(TEST_INSTANCE, "Master", - "instance1", "Slave", "instance2", "Slave", "instance3", "Slave")); - doReturn(externalView).when(mock.dataAccessor) - .getProperty(argThat(new PropertyKeyArgument(PropertyType.EXTERNALVIEW))); - StateModelDefinition stateModelDefinition = mock(StateModelDefinition.class); - when(stateModelDefinition.getInitialState()).thenReturn("OFFLINE"); - doReturn(stateModelDefinition).when(mock.dataAccessor) - .getProperty(argThat(new PropertyKeyArgument(PropertyType.STATEMODELDEFS))); - - // Set instance operation to be EVACUATE for instance2 - InstanceConfig instanceConfig2 = new InstanceConfig("instance2"); - InstanceConfig instanceConfig3 = new InstanceConfig("instance3"); - instanceConfig2.setInstanceOperation(InstanceConstants.InstanceOperation.EVACUATE); - doReturn(instanceConfig2).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("instance2")); - doReturn(instanceConfig3).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("instance3")); - - // Add instance1 to toBeStoppedInstances - Set toBeStoppedInstances = new HashSet<>(); - toBeStoppedInstances.add("instance1"); - boolean result = - InstanceValidationUtil.siblingNodesActiveReplicaCheck(mock.dataAccessor, TEST_INSTANCE, toBeStoppedInstances); - Assert.assertFalse(result); - - // Remove instance1 from toBeStoppedInstances and add its config - InstanceConfig instanceConfig1 = new InstanceConfig("instance1"); - doReturn(instanceConfig1).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("instance1")); - toBeStoppedInstances = new HashSet<>(); - result = - InstanceValidationUtil.siblingNodesActiveReplicaCheck(mock.dataAccessor, TEST_INSTANCE, toBeStoppedInstances); - Assert.assertTrue(result); - } - @Test public void TestSiblingNodesActiveReplicaCheck_fail() { String resource = "resource"; @@ -568,9 +504,6 @@ public void TestSiblingNodesActiveReplicaCheck_fail() { doReturn(stateModelDefinition).when(mock.dataAccessor) .getProperty(argThat(new PropertyKeyArgument(PropertyType.STATEMODELDEFS))); - // set default instance config - setDefaultInstanceConfigs(mock); - boolean result = InstanceValidationUtil.siblingNodesActiveReplicaCheck(mock.dataAccessor, TEST_INSTANCE); @@ -619,19 +552,6 @@ public void TestSiblingNodesActiveReplicaCheck_exception_whenExternalViewUnavail InstanceValidationUtil.siblingNodesActiveReplicaCheck(mock.dataAccessor, TEST_INSTANCE); } - private void setDefaultInstanceConfigs(Mock mock) { - // set default instance config - InstanceConfig instanceConfig1 = new InstanceConfig("instance1"); - doReturn(instanceConfig1).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("instance1")); - InstanceConfig instanceConfig2 = new InstanceConfig("instance2"); - doReturn(instanceConfig2).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("instance2")); - InstanceConfig instanceConfig3 = new InstanceConfig("instance3"); - doReturn(instanceConfig3).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("instance3")); - } - private class Mock { HelixDataAccessor dataAccessor; ConfigAccessor configAccessor; diff --git a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java index 0f617eea97..d44718cf3d 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementService.java @@ -148,24 +148,26 @@ public MaintenanceManagementService(ZKHelixDataAccessor dataAccessor, _namespace = namespace; } - @VisibleForTesting - MaintenanceManagementService(MaintenanceManagementServiceBuilder builder) { + private MaintenanceManagementService(ZKHelixDataAccessor dataAccessor, + ConfigAccessor configAccessor, CustomRestClient customRestClient, boolean skipZKRead, + boolean continueOnFailure, Set skipHealthCheckCategories, + List stoppableHealthCheckList, String namespace) { _dataAccessor = - new HelixDataAccessorWrapper(builder.getDataAccessor(), builder.getCustomRestClient(), - builder.getNamespace()); - _configAccessor = builder.getConfigAccessor(); - _customRestClient = builder.getCustomRestClient(); - _skipZKRead = builder.isSkipZKRead(); + new HelixDataAccessorWrapper(dataAccessor, customRestClient, + namespace); + _configAccessor = configAccessor; + _customRestClient = customRestClient; + _skipZKRead = skipZKRead; _nonBlockingHealthChecks = - builder.isContinueOnFailure() ? Collections.singleton(ALL_HEALTH_CHECK_NONBLOCK) + continueOnFailure ? Collections.singleton(ALL_HEALTH_CHECK_NONBLOCK) : Collections.emptySet(); _skipHealthCheckCategories = - builder.getSkipHealthCheckCategories() == null ? Collections.emptySet() - : builder.getSkipHealthCheckCategories(); + skipHealthCheckCategories == null ? Collections.emptySet() + : skipHealthCheckCategories; _stoppableHealthCheckList = - builder.getStoppableHealthCheckList() == null ? Collections.emptyList() - : builder.getStoppableHealthCheckList(); - _namespace = builder.getNamespace(); + stoppableHealthCheckList == null ? Collections.emptyList() + : stoppableHealthCheckList; + _namespace = namespace; } /** @@ -795,4 +797,86 @@ protected Map getInstanceHealthStatus(String clusterId, String return healthStatus; } + + public static class MaintenanceManagementServiceBuilder { + private ConfigAccessor _configAccessor; + private boolean _skipZKRead; + private String _namespace; + private ZKHelixDataAccessor _dataAccessor; + private CustomRestClient _customRestClient; + private boolean _continueOnFailure; + private Set _skipHealthCheckCategories = Collections.emptySet(); + private List _stoppableHealthCheckList = Collections.emptyList(); + + public MaintenanceManagementServiceBuilder setConfigAccessor(ConfigAccessor configAccessor) { + _configAccessor = configAccessor; + return this; + } + + public MaintenanceManagementServiceBuilder setSkipZKRead(boolean skipZKRead) { + _skipZKRead = skipZKRead; + return this; + } + + public MaintenanceManagementServiceBuilder setNamespace(String namespace) { + _namespace = namespace; + return this; + } + + public MaintenanceManagementServiceBuilder setDataAccessor( + ZKHelixDataAccessor dataAccessor) { + _dataAccessor = dataAccessor; + return this; + } + + public MaintenanceManagementServiceBuilder setCustomRestClient( + CustomRestClient customRestClient) { + _customRestClient = customRestClient; + return this; + } + + public MaintenanceManagementServiceBuilder setContinueOnFailure(boolean continueOnFailure) { + _continueOnFailure = continueOnFailure; + return this; + } + + public MaintenanceManagementServiceBuilder setSkipHealthCheckCategories( + Set skipHealthCheckCategories) { + _skipHealthCheckCategories = skipHealthCheckCategories; + return this; + } + + public MaintenanceManagementServiceBuilder setStoppableHealthCheckList( + List stoppableHealthCheckList) { + _stoppableHealthCheckList = stoppableHealthCheckList; + return this; + } + + public MaintenanceManagementService build() { + validate(); + return new MaintenanceManagementService(_dataAccessor, _configAccessor, _customRestClient, + _skipZKRead, _continueOnFailure, _skipHealthCheckCategories, _stoppableHealthCheckList, + _namespace); + } + + private void validate() throws IllegalArgumentException { + List msg = new ArrayList<>(); + if (_configAccessor == null) { + msg.add("'configAccessor' can't be null."); + } + if (_namespace == null) { + msg.add("'namespace' can't be null."); + } + if (_dataAccessor == null) { + msg.add("'_dataAccessor' can't be null."); + } + if (_customRestClient == null) { + msg.add("'customRestClient' can't be null."); + } + if (msg.size() != 0) { + throw new IllegalArgumentException( + "One or more mandatory arguments are not set " + msg); + } + } + } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementServiceBuilder.java b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementServiceBuilder.java deleted file mode 100644 index 9d4687e850..0000000000 --- a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/MaintenanceManagementServiceBuilder.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.apache.helix.rest.clusterMaintenanceService; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import org.apache.helix.ConfigAccessor; -import org.apache.helix.manager.zk.ZKHelixDataAccessor; -import org.apache.helix.rest.client.CustomRestClient; -import org.apache.helix.rest.server.json.instance.StoppableCheck; - - -public class MaintenanceManagementServiceBuilder { - private ConfigAccessor _configAccessor; - private boolean _skipZKRead; - private String _namespace; - private ZKHelixDataAccessor _dataAccessor; - private CustomRestClient _customRestClient; - private boolean _continueOnFailure; - private Set _skipHealthCheckCategories = Collections.emptySet(); - private List _stoppableHealthCheckList = Collections.emptyList(); - - public ConfigAccessor getConfigAccessor() { - return _configAccessor; - } - - public boolean isSkipZKRead() { - return _skipZKRead; - } - - public String getNamespace() { - return _namespace; - } - - public ZKHelixDataAccessor getDataAccessor() { - return _dataAccessor; - } - - public CustomRestClient getCustomRestClient() { - return _customRestClient; - } - - public boolean isContinueOnFailure() { - return _continueOnFailure; - } - - public Set getSkipHealthCheckCategories() { - return _skipHealthCheckCategories; - } - - public List getStoppableHealthCheckList() { - return _stoppableHealthCheckList; - } - - public MaintenanceManagementServiceBuilder setConfigAccessor(ConfigAccessor configAccessor) { - _configAccessor = configAccessor; - return this; - } - - public MaintenanceManagementServiceBuilder setSkipZKRead(boolean skipZKRead) { - _skipZKRead = skipZKRead; - return this; - } - - public MaintenanceManagementServiceBuilder setNamespace(String namespace) { - _namespace = namespace; - return this; - } - - public MaintenanceManagementServiceBuilder setDataAccessor( - ZKHelixDataAccessor dataAccessor) { - _dataAccessor = dataAccessor; - return this; - } - - public MaintenanceManagementServiceBuilder setCustomRestClient( - CustomRestClient customRestClient) { - _customRestClient = customRestClient; - return this; - } - - public MaintenanceManagementServiceBuilder setContinueOnFailure(boolean continueOnFailure) { - _continueOnFailure = continueOnFailure; - return this; - } - - public MaintenanceManagementServiceBuilder setSkipHealthCheckCategories( - Set skipHealthCheckCategories) { - _skipHealthCheckCategories = skipHealthCheckCategories; - return this; - } - - public MaintenanceManagementServiceBuilder setStoppableHealthCheckList( - List stoppableHealthCheckList) { - _stoppableHealthCheckList = stoppableHealthCheckList; - return this; - } - - public MaintenanceManagementService createMaintenanceManagementService() { - if (_configAccessor == null || _namespace == null || _dataAccessor == null - || _customRestClient == null) { - throw new IllegalArgumentException( - "One or more of following mandatory arguments are not set: '_configAccessor', " - + "'_namespace', '_dataAccessor', '_customRestClient'."); - } - return new MaintenanceManagementService(this); - } -} \ No newline at end of file diff --git a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java index 25036bed8e..647a8c1eca 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/clusterMaintenanceService/StoppableInstancesSelector.java @@ -34,6 +34,10 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.helix.PropertyKey; +import org.apache.helix.constants.InstanceConstants; +import org.apache.helix.manager.zk.ZKHelixDataAccessor; +import org.apache.helix.model.InstanceConfig; import org.apache.helix.rest.server.json.cluster.ClusterTopology; import org.apache.helix.rest.server.json.instance.StoppableCheck; import org.apache.helix.rest.server.resources.helix.InstancesAccessor; @@ -48,15 +52,17 @@ public class StoppableInstancesSelector { private final String _customizedInput; private final MaintenanceManagementService _maintenanceService; private final ClusterTopology _clusterTopology; + private final ZKHelixDataAccessor _dataAccessor; - public StoppableInstancesSelector(String clusterId, List orderOfZone, + private StoppableInstancesSelector(String clusterId, List orderOfZone, String customizedInput, MaintenanceManagementService maintenanceService, - ClusterTopology clusterTopology) { + ClusterTopology clusterTopology, ZKHelixDataAccessor dataAccessor) { _clusterId = clusterId; _orderOfZone = orderOfZone; _customizedInput = customizedInput; _maintenanceService = maintenanceService; _clusterTopology = clusterTopology; + _dataAccessor = dataAccessor; } /** @@ -81,6 +87,7 @@ public ObjectNode getStoppableInstancesInSingleZone(List instances, ObjectNode failedStoppableInstances = result.putObject( InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); Set toBeStoppedInstancesSet = new HashSet<>(toBeStoppedInstances); + collectEvacuatingInstances(toBeStoppedInstancesSet); List zoneBasedInstance = getZoneBasedInstances(instances, _clusterTopology.toZoneMapping()); @@ -112,6 +119,7 @@ public ObjectNode getStoppableInstancesCrossZones(List instances, ObjectNode failedStoppableInstances = result.putObject( InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); Set toBeStoppedInstancesSet = new HashSet<>(toBeStoppedInstances); + collectEvacuatingInstances(toBeStoppedInstancesSet); Map> zoneMapping = _clusterTopology.toZoneMapping(); for (String zone : _orderOfZone) { @@ -249,12 +257,31 @@ private Map> getOrderedZoneToInstancesMap( (existing, replacement) -> existing, LinkedHashMap::new)); } + /** + * Collect instances marked for evacuation in the current topology and add them into given the set + * + * @param toBeStoppedInstances A set of instances we presume to be stopped. + */ + private void collectEvacuatingInstances(Set toBeStoppedInstances) { + Set allInstances = _clusterTopology.getAllInstances(); + for (String instance : allInstances) { + PropertyKey.Builder propertyKeyBuilder = _dataAccessor.keyBuilder(); + InstanceConfig instanceConfig = + _dataAccessor.getProperty(propertyKeyBuilder.instanceConfig(instance)); + if (InstanceConstants.InstanceOperation.EVACUATE.name() + .equals(instanceConfig.getInstanceOperation())) { + toBeStoppedInstances.add(instance); + } + } + } + public static class StoppableInstancesSelectorBuilder { private String _clusterId; private List _orderOfZone; private String _customizedInput; private MaintenanceManagementService _maintenanceService; private ClusterTopology _clusterTopology; + private ZKHelixDataAccessor _dataAccessor; public StoppableInstancesSelectorBuilder setClusterId(String clusterId) { _clusterId = clusterId; @@ -282,9 +309,14 @@ public StoppableInstancesSelectorBuilder setClusterTopology(ClusterTopology clus return this; } + public StoppableInstancesSelectorBuilder setDataAccessor(ZKHelixDataAccessor dataAccessor) { + _dataAccessor = dataAccessor; + return this; + } + public StoppableInstancesSelector build() { return new StoppableInstancesSelector(_clusterId, _orderOfZone, _customizedInput, - _maintenanceService, _clusterTopology); + _maintenanceService, _clusterTopology, _dataAccessor); } } } diff --git a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java index 2f40c6cf0d..fded7e6f9e 100644 --- a/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java +++ b/helix-rest/src/main/java/org/apache/helix/rest/server/resources/helix/InstancesAccessor.java @@ -40,7 +40,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableList; import org.apache.helix.HelixAdmin; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; @@ -50,7 +49,6 @@ import org.apache.helix.rest.client.CustomRestClientFactory; import org.apache.helix.rest.clusterMaintenanceService.HealthCheck; import org.apache.helix.rest.clusterMaintenanceService.MaintenanceManagementService; -import org.apache.helix.rest.clusterMaintenanceService.MaintenanceManagementServiceBuilder; import org.apache.helix.rest.common.HttpConstants; import org.apache.helix.rest.clusterMaintenanceService.StoppableInstancesSelector; import org.apache.helix.rest.server.filters.ClusterAuth; @@ -67,11 +65,7 @@ @Path("/clusters/{clusterId}/instances") public class InstancesAccessor extends AbstractHelixResource { private final static Logger _logger = LoggerFactory.getLogger(InstancesAccessor.class); - // This parameter indicates the users would like perform all existing helix checks on the - // given instances list. - private final static String SELECT_ALL_STOPPABLE_CHECKS = "ALL"; - private final static List ALL_STOPPABLE_CHECK_LIST = - ImmutableList.of(SELECT_ALL_STOPPABLE_CHECKS); + public enum InstancesProperties { instances, online, @@ -238,7 +232,9 @@ private Response batchGetStoppableInstances(String clusterId, JsonNode node, boo List orderOfZone = null; String customizedInput = null; List toBeStoppedInstances = Collections.emptyList(); - List stoppableCheckList = null; + // By default, if stoppable_check_list is unset, all checks are performed to maintain + // backward compatibility with existing clients. + List stoppableCheckList = HealthCheck.STOPPABLE_CHECK_LIST; if (node.get(InstancesAccessor.InstancesProperties.customized_values.name()) != null) { customizedInput = node.get(InstancesAccessor.InstancesProperties.customized_values.name()).toString(); @@ -275,30 +271,21 @@ private Response batchGetStoppableInstances(String clusterId, JsonNode node, boo List list = OBJECT_MAPPER.readValue( node.get(InstancesProperties.stoppable_check_list.name()).toString(), OBJECT_MAPPER.getTypeFactory().constructCollectionType(List.class, String.class)); - - if (ALL_STOPPABLE_CHECK_LIST.equals(list)) { - stoppableCheckList = HealthCheck.STOPPABLE_CHECK_LIST; - } else { - try { - stoppableCheckList = - list.stream().map(HealthCheck::valueOf).collect(Collectors.toList()); - } catch (IllegalArgumentException e) { - String message = - "'stoppable_check_list' has invalid check names: " + list - + ". Supported checks: " + HealthCheck.STOPPABLE_CHECK_LIST; - _logger.error(message, e); - return badRequest(message); - } + try { + stoppableCheckList = + list.stream().map(HealthCheck::valueOf).collect(Collectors.toList()); + } catch (IllegalArgumentException e) { + String message = + "'stoppable_check_list' has invalid check names: " + list + + ". Supported checks: " + HealthCheck.STOPPABLE_CHECK_LIST; + _logger.error(message, e); + return badRequest(message); } - } else { - // By default, if stoppable_check_list is unset, all checks are performed to maintain - // backward compatibility with existing clients. - stoppableCheckList = HealthCheck.STOPPABLE_CHECK_LIST; } String namespace = getNamespace(); MaintenanceManagementService maintenanceService = - new MaintenanceManagementServiceBuilder() + new MaintenanceManagementService.MaintenanceManagementServiceBuilder() .setDataAccessor((ZKHelixDataAccessor) getDataAccssor(clusterId)) .setConfigAccessor(getConfigAccessor()) .setSkipZKRead(skipZKRead) @@ -307,7 +294,7 @@ private Response batchGetStoppableInstances(String clusterId, JsonNode node, boo .setSkipHealthCheckCategories(skipHealthCheckCategories) .setNamespace(namespace) .setStoppableHealthCheckList(stoppableCheckList) - .createMaintenanceManagementService(); + .build(); ClusterService clusterService = new ClusterServiceImpl(getDataAccssor(clusterId), getConfigAccessor()); @@ -319,6 +306,7 @@ private Response batchGetStoppableInstances(String clusterId, JsonNode node, boo .setCustomizedInput(customizedInput) .setMaintenanceService(maintenanceService) .setClusterTopology(clusterTopology) + .setDataAccessor((ZKHelixDataAccessor) getDataAccssor(clusterId)) .build(); stoppableInstancesSelector.calculateOrderOfZone(instances, random); ObjectNode result; diff --git a/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java b/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java index 2b530ff733..a49a95066f 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/clusterMaintenanceService/TestMaintenanceManagementService.java @@ -39,7 +39,6 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor; import org.apache.helix.model.ExternalView; import org.apache.helix.model.IdealState; -import org.apache.helix.model.InstanceConfig; import org.apache.helix.model.LeaderStandbySMD; import org.apache.helix.model.MasterSlaveSMD; import org.apache.helix.model.RESTConfig; @@ -61,7 +60,6 @@ import static org.mockito.Matchers.anyList; import static org.mockito.Matchers.anyMap; import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -221,8 +219,7 @@ public void testGetInstanceStoppableCheckWhenCustomInstanceCheckDisabled() throw when(_dataAccessorWrapper.getAllPartitionsHealthOnLiveInstance(any(), anyMap(), anyBoolean())).thenReturn(Collections.emptyMap()); when(_dataAccessorWrapper.getProperty((PropertyKey) any())).thenReturn(new LeaderStandbySMD()); - PropertyKey.Builder builder = new PropertyKey.Builder(TEST_CLUSTER); - when(_dataAccessorWrapper.keyBuilder()).thenReturn(builder); + when(_dataAccessorWrapper.keyBuilder()).thenReturn(new PropertyKey.Builder(TEST_CLUSTER)); when(_dataAccessorWrapper.getChildValues(any(), anyBoolean())).thenReturn( Arrays.asList(externalView)); MockMaintenanceManagementService service = @@ -231,14 +228,6 @@ public void testGetInstanceStoppableCheckWhenCustomInstanceCheckDisabled() throw new HashSet<>(Arrays.asList(StoppableCheck.Category.CUSTOM_INSTANCE_CHECK)), HelixRestNamespace.DEFAULT_NAMESPACE_NAME); - // set default instance config - InstanceConfig instanceConfig1 = new InstanceConfig(TEST_INSTANCE); - doReturn(instanceConfig1).when(_dataAccessorWrapper) - .getProperty(builder.instanceConfig(TEST_INSTANCE)); - InstanceConfig instanceConfig2 = new InstanceConfig("sibling_instance"); - doReturn(instanceConfig2).when(_dataAccessorWrapper) - .getProperty(builder.instanceConfig("sibling_instance")); - StoppableCheck actual = service.getInstanceStoppableCheck(TEST_CLUSTER, TEST_INSTANCE, ""); List expectedFailedChecks = Arrays.asList( StoppableCheck.Category.CUSTOM_PARTITION_CHECK.getPrefix() @@ -322,14 +311,6 @@ public void testCustomPartitionCheckWithSkipZKRead() throws IOException { when(mockAccessor.get(zkHelixDataAccessor.keyBuilder().stateModelDef("MasterSlave").getPath(), new Stat(), AccessOption.PERSISTENT)).thenReturn(MasterSlaveSMD.build().getRecord()); - // set default instance config - InstanceConfig instanceConfig1 = new InstanceConfig(TEST_INSTANCE); - when(mockAccessor.get(zkHelixDataAccessor.keyBuilder().instanceConfig(TEST_INSTANCE).getPath(), - new Stat(), AccessOption.PERSISTENT)).thenReturn(instanceConfig1.getRecord()); - InstanceConfig instanceConfig2 = new InstanceConfig(siblingInstance); - when(mockAccessor.get(zkHelixDataAccessor.keyBuilder().instanceConfig(siblingInstance).getPath(), - new Stat(), AccessOption.PERSISTENT)).thenReturn(instanceConfig2.getRecord()); - // Valid data only from ZK, pass the check MockMaintenanceManagementService instanceServiceReadZK = new MockMaintenanceManagementService(zkHelixDataAccessor, _configAccessor, _customRestClient, false, diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java index 089bac12d8..74bdfdec65 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/TestInstancesAccessor.java @@ -34,6 +34,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.helix.TestHelper; +import org.apache.helix.constants.InstanceConstants; import org.apache.helix.model.ClusterConfig; import org.apache.helix.model.InstanceConfig; import org.apache.helix.rest.server.resources.helix.InstancesAccessor; @@ -113,7 +114,7 @@ public void testInstanceStoppableZoneBasedWithToBeStoppedInstances() throws IOEx System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test + @Test(dependsOnMethods = "testInstanceStoppableZoneBasedWithToBeStoppedInstances") public void testInstanceStoppableZoneBasedWithoutZoneOrder() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); String content = String.format( @@ -144,7 +145,8 @@ public void testInstanceStoppableZoneBasedWithoutZoneOrder() throws IOException System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test(dataProvider = "generatePayloadCrossZoneStoppableCheckWithZoneOrder") + @Test(dataProvider = "generatePayloadCrossZoneStoppableCheckWithZoneOrder", + dependsOnMethods = "testInstanceStoppableZoneBasedWithoutZoneOrder") public void testCrossZoneStoppableWithZoneOrder(String content) throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); Response response = new JerseyUriRequestBuilder( @@ -166,7 +168,7 @@ public void testCrossZoneStoppableWithZoneOrder(String content) throws IOExcepti System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test + @Test(dependsOnMethods = "testCrossZoneStoppableWithZoneOrder") public void testCrossZoneStoppableWithoutZoneOrder() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); String content = String.format( @@ -199,8 +201,8 @@ public void testCrossZoneStoppableWithoutZoneOrder() throws IOException { System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test - public void testInstanceStoppable_crosszonebased_with_selectedchecklist() throws IOException { + @Test(dependsOnMethods = "testCrossZoneStoppableWithoutZoneOrder") + public void testInstanceStoppableCrossZoneBasedWithSelectedCheckList() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); // Select instances with cross zone based and perform all checks String content = @@ -209,29 +211,12 @@ public void testInstanceStoppable_crosszonebased_with_selectedchecklist() throws InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(), InstancesAccessor.InstancesProperties.instances.name(), "instance0", "instance1", "instance2", "instance3", "instance4", "instance5", "invalidInstance", - InstancesAccessor.InstancesProperties.stoppable_check_list.name(), "ALL"); - Response response = - new JerseyUriRequestBuilder("clusters/{}/instances?command=stoppable").format( - STOPPABLE_CLUSTER).post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); - JsonNode jsonNode = OBJECT_MAPPER.readTree(response.readEntity(String.class)); - Assert.assertFalse( - jsonNode.withArray(InstancesAccessor.InstancesProperties.instance_stoppable_parallel.name()) - .elements().hasNext()); - JsonNode nonStoppableInstances = jsonNode.get( - InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); - Assert.assertEquals(getStringSet(nonStoppableInstances, "instance0"), - ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED")); - Assert.assertEquals(getStringSet(nonStoppableInstances, "instance1"), - ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT", "HELIX:INSTANCE_NOT_ENABLED", - "HELIX:INSTANCE_NOT_STABLE")); - Assert.assertEquals(getStringSet(nonStoppableInstances, "instance2"), - ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED")); - Assert.assertEquals(getStringSet(nonStoppableInstances, "instance3"), - ImmutableSet.of("HELIX:HAS_DISABLED_PARTITION", "HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED")); - Assert.assertEquals(getStringSet(nonStoppableInstances, "instance4"), - ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT", "HELIX:INSTANCE_NOT_ALIVE", - "HELIX:INSTANCE_NOT_STABLE")); - Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"), ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST")); + InstancesAccessor.InstancesProperties.stoppable_check_list.name(), "DUMMY_TEST_NO_EXISTS"); + + new JerseyUriRequestBuilder("clusters/{}/instances?command=stoppable").format(STOPPABLE_CLUSTER) + .isBodyReturnExpected(true) + .expectedReturnStatusCode(Response.Status.BAD_REQUEST.getStatusCode()) + .post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); // Select instances with cross zone based and perform a subset of checks content = String.format( @@ -242,11 +227,11 @@ public void testInstanceStoppable_crosszonebased_with_selectedchecklist() throws "instance2", "instance3", "instance4", "instance5", "invalidInstance", InstancesAccessor.InstancesProperties.zone_order.name(), "zone2", "zone1", InstancesAccessor.InstancesProperties.stoppable_check_list.name(), "EMPTY_RESOURCE_ASSIGNMENT", "INSTANCE_NOT_ALIVE"); - response = new JerseyUriRequestBuilder( + Response response = new JerseyUriRequestBuilder( "clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format( STOPPABLE_CLUSTER).post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); - jsonNode = OBJECT_MAPPER.readTree(response.readEntity(String.class)); - nonStoppableInstances = jsonNode.get( + JsonNode jsonNode = OBJECT_MAPPER.readTree(response.readEntity(String.class)); + JsonNode nonStoppableInstances = jsonNode.get( InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); Assert.assertEquals(getStringSet(nonStoppableInstances, "instance5"), ImmutableSet.of("HELIX:EMPTY_RESOURCE_ASSIGNMENT", "HELIX:INSTANCE_NOT_ALIVE")); @@ -260,7 +245,53 @@ public void testInstanceStoppable_crosszonebased_with_selectedchecklist() throws System.out.println("End test :" + TestHelper.getTestMethodName()); } - @Test(dependsOnMethods = "testInstanceStoppableZoneBasedWithToBeStoppedInstances") + @Test(dependsOnMethods = "testInstanceStoppableCrossZoneBasedWithSelectedCheckList") + public void testInstanceStoppableCrossZoneBasedWithEvacuatingInstances() throws IOException { + System.out.println("Start test :" + TestHelper.getTestMethodName()); + String content = String.format( + "{\"%s\":\"%s\",\"%s\":[\"%s\",\"%s\",\"%s\",\"%s\", \"%s\", \"%s\", \"%s\",\"%s\", \"%s\", \"%s\"]}", + InstancesAccessor.InstancesProperties.selection_base.name(), + InstancesAccessor.InstanceHealthSelectionBase.cross_zone_based.name(), + InstancesAccessor.InstancesProperties.instances.name(), "instance1", "instance3", + "instance6", "instance9", "instance10", "instance11", "instance12", "instance13", + "instance14", "invalidInstance"); + + // Change instance config of instance1 & instance0 to be evacuating + String instance0 = "instance0"; + InstanceConfig instanceConfig = _configAccessor.getInstanceConfig(STOPPABLE_CLUSTER2, instance0); + instanceConfig.setInstanceOperation(InstanceConstants.InstanceOperation.EVACUATE); + _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance0, instanceConfig); + String instance1 = "instance1"; + InstanceConfig instanceConfig1 = _configAccessor.getInstanceConfig(STOPPABLE_CLUSTER2, instance1); + instanceConfig1.setInstanceOperation(InstanceConstants.InstanceOperation.EVACUATE); + _configAccessor.setInstanceConfig(STOPPABLE_CLUSTER2, instance1, instanceConfig1); + // It takes time to reflect the changes. + BestPossibleExternalViewVerifier verifier = + new BestPossibleExternalViewVerifier.Builder(STOPPABLE_CLUSTER2).setZkAddr(ZK_ADDR).build(); + Assert.assertTrue(verifier.verifyByPolling()); + + Response response = new JerseyUriRequestBuilder( + "clusters/{}/instances?command=stoppable&skipHealthCheckCategories=CUSTOM_INSTANCE_CHECK,CUSTOM_PARTITION_CHECK").format( + STOPPABLE_CLUSTER2).post(this, Entity.entity(content, MediaType.APPLICATION_JSON_TYPE)); + JsonNode jsonNode = OBJECT_MAPPER.readTree(response.readEntity(String.class)); + + Set stoppableSet = getStringSet(jsonNode, + InstancesAccessor.InstancesProperties.instance_stoppable_parallel.name()); + Assert.assertTrue(stoppableSet.contains("instance12") + && stoppableSet.contains("instance11") && stoppableSet.contains("instance10")); + + JsonNode nonStoppableInstances = jsonNode.get( + InstancesAccessor.InstancesProperties.instance_not_stoppable_with_reasons.name()); + Assert.assertEquals(getStringSet(nonStoppableInstances, "instance13"), + ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED")); + Assert.assertEquals(getStringSet(nonStoppableInstances, "instance14"), + ImmutableSet.of("HELIX:MIN_ACTIVE_REPLICA_CHECK_FAILED")); + Assert.assertEquals(getStringSet(nonStoppableInstances, "invalidInstance"), + ImmutableSet.of("HELIX:INSTANCE_NOT_EXIST")); + System.out.println("End test :" + TestHelper.getTestMethodName()); + } + + @Test(dependsOnMethods = "testInstanceStoppableCrossZoneBasedWithEvacuatingInstances") public void testInstanceStoppable_zoneBased_zoneOrder() throws IOException { System.out.println("Start test :" + TestHelper.getTestMethodName()); // Select instances with zone based diff --git a/helix-rest/src/test/java/org/apache/helix/rest/server/util/TestInstanceValidationUtilInRest.java b/helix-rest/src/test/java/org/apache/helix/rest/server/util/TestInstanceValidationUtilInRest.java index 5754e77450..836e6bd645 100644 --- a/helix-rest/src/test/java/org/apache/helix/rest/server/util/TestInstanceValidationUtilInRest.java +++ b/helix-rest/src/test/java/org/apache/helix/rest/server/util/TestInstanceValidationUtilInRest.java @@ -30,21 +30,18 @@ import org.apache.helix.HelixDataAccessor; import org.apache.helix.PropertyKey; import org.apache.helix.model.ExternalView; -import org.apache.helix.model.InstanceConfig; import org.apache.helix.model.MasterSlaveSMD; import org.apache.helix.model.StateModelDefinition; import org.apache.helix.util.InstanceValidationUtil; import org.junit.Assert; import org.testng.annotations.Test; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TestInstanceValidationUtilInRest{ private static final String RESOURCE_NAME = "TestResource"; private static final String TEST_CLUSTER = "TestCluster"; - private static final PropertyKey.Builder BUILDER = new PropertyKey.Builder(TEST_CLUSTER); @Test public void testPartitionLevelCheck() { @@ -52,11 +49,13 @@ public void testPartitionLevelCheck() { Mock mock = new Mock(); HelixDataAccessor accessor = mock.dataAccessor; - when(mock.dataAccessor.keyBuilder()).thenReturn(BUILDER); - when(mock.dataAccessor.getProperty(BUILDER.stateModelDef(MasterSlaveSMD.name))) + when(mock.dataAccessor.keyBuilder()) + .thenReturn(new PropertyKey.Builder(TEST_CLUSTER)); + when(mock.dataAccessor + .getProperty(new PropertyKey.Builder(TEST_CLUSTER).stateModelDef(MasterSlaveSMD.name))) .thenReturn(mock.stateModel); when(mock.stateModel.getTopState()).thenReturn("MASTER"); - setDefaultInstanceConfigs(mock); + Map> failedPartitions = InstanceValidationUtil .perPartitionHealthCheck(externalViews, preparePartitionStateMap(), "h2", accessor); @@ -70,12 +69,14 @@ public void testPartitionLevelCheckInitState() { Mock mock = new Mock(); HelixDataAccessor accessor = mock.dataAccessor; - when(mock.dataAccessor.keyBuilder()).thenReturn(BUILDER); - when(mock.dataAccessor.getProperty(BUILDER.stateModelDef(MasterSlaveSMD.name))) + when(mock.dataAccessor.keyBuilder()) + .thenReturn(new PropertyKey.Builder(TEST_CLUSTER)); + when(mock.dataAccessor + .getProperty(new PropertyKey.Builder(TEST_CLUSTER).stateModelDef(MasterSlaveSMD.name))) .thenReturn(mock.stateModel); when(mock.stateModel.getTopState()).thenReturn("MASTER"); when(mock.stateModel.getInitialState()).thenReturn("OFFLINE"); - setDefaultInstanceConfigs(mock); + Map> partitionStateMap = new HashMap<>(); partitionStateMap.put("h1", new HashMap<>()); @@ -112,12 +113,13 @@ public void testPartitionLevelCheckWithToBeStoppedNode() { Mock mock = new Mock(); HelixDataAccessor accessor = mock.dataAccessor; - when(mock.dataAccessor.keyBuilder()).thenReturn(BUILDER); - when(mock.dataAccessor.getProperty(BUILDER.stateModelDef(MasterSlaveSMD.name))) + when(mock.dataAccessor.keyBuilder()) + .thenReturn(new PropertyKey.Builder(TEST_CLUSTER)); + when(mock.dataAccessor + .getProperty(new PropertyKey.Builder(TEST_CLUSTER).stateModelDef(MasterSlaveSMD.name))) .thenReturn(mock.stateModel); when(mock.stateModel.getTopState()).thenReturn("MASTER"); when(mock.stateModel.getInitialState()).thenReturn("OFFLINE"); - setDefaultInstanceConfigs(mock); Map> partitionStateMap = new HashMap<>(); partitionStateMap.put("h1", new HashMap<>()); @@ -227,21 +229,6 @@ private ExternalView prepareExternalViewOnline() { return externalView; } - private void setDefaultInstanceConfigs(Mock mock) { - InstanceConfig instanceConfig1 = new InstanceConfig("h1"); - doReturn(instanceConfig1).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("h1")); - InstanceConfig instanceConfig2 = new InstanceConfig("h2"); - doReturn(instanceConfig2).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("h2")); - InstanceConfig instanceConfig3 = new InstanceConfig("h3"); - doReturn(instanceConfig3).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("h3")); - InstanceConfig instanceConfig4 = new InstanceConfig("h4"); - doReturn(instanceConfig4).when(mock.dataAccessor) - .getProperty(BUILDER.instanceConfig("h4")); - } - private final class Mock { private HelixDataAccessor dataAccessor = mock(HelixDataAccessor.class); private StateModelDefinition stateModel = mock(StateModelDefinition.class);