diff --git a/.github/workflows/helix-front.yml b/.github/workflows/helix-front.yml index 961aa0a4ab..9b67dc33e1 100644 --- a/.github/workflows/helix-front.yml +++ b/.github/workflows/helix-front.yml @@ -1,9 +1,9 @@ name: Helix Front CI on: pull_request: - branches: [ master ] + branches: [master] paths: - - 'helix-front/**' + - "helix-front/**" jobs: CI: @@ -16,9 +16,9 @@ jobs: - name: Setup Node environment uses: actions/setup-node@v3 with: - node-version: '16.x' - cache: 'yarn' - cache-dependency-path: 'helix-front/yarn.lock' + node-version: "14.x" + cache: "yarn" + cache-dependency-path: "helix-front/yarn.lock" - name: Install dependencies run: yarn diff --git a/helix-core/pom.xml b/helix-core/pom.xml index d3934f579d..ba3d7b7e51 100644 --- a/helix-core/pom.xml +++ b/helix-core/pom.xml @@ -179,14 +179,52 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.12.1 + + + JDK 8 + compile + + compile + + + ${project.build.outputDirectory}_jdk8 + 8 + true + + + + JDK 11 + compile + + compile + + + 11 + true + + + + org.apache.maven.plugins maven-jar-plugin + 3.3.0 + default-package-jdk11 + package + jar test-jar + + ${project.build.outputDirectory}_jdk8 + jdk8 + diff --git a/helix-core/src/main/java/org/apache/helix/BaseDataAccessor.java b/helix-core/src/main/java/org/apache/helix/BaseDataAccessor.java index 162ffb05d2..4fafb42717 100644 --- a/helix-core/src/main/java/org/apache/helix/BaseDataAccessor.java +++ b/helix-core/src/main/java/org/apache/helix/BaseDataAccessor.java @@ -112,6 +112,17 @@ default boolean multiSet(Map> updaterByPath) { */ boolean remove(String path, int options); + /** + * This will remove the ZNode, if the ZNode's version matches the provided expectedVersion. This + * operation will fail if the node has any children. + * @param path Path to the ZNode to update + * @param options Set the type of ZNode see the valid values in {@link AccessOption} + * @param expectedVersion the expected version of the node to be removed, -1 means match any + * version + * @return true if the removal succeeded, false otherwise + */ + boolean removeWithExpectedVersion(String path, int options, int expectedVersion); + /** * Use it when creating children under a parent node. This will use async api for better * performance. If the child already exists it will return false. diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java index 9f04de5350..9b5b4f9293 100644 --- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java +++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java @@ -28,6 +28,7 @@ import org.apache.helix.controller.stages.CurrentStateOutput; import org.apache.helix.model.IdealState; import org.apache.helix.model.Partition; +import org.apache.helix.model.StateModelDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,12 +56,38 @@ public IdealState computeNewIdealState(String resourceName, IdealState currentId // One principal is to prohibit DROP -> OFFLINE and OFFLINE -> DROP state transitions. // Derived preference list from current state with state priority + StateModelDefinition stateModelDef = clusterData.getStateModelDef(currentIdealState.getStateModelDefRef()); + for (Partition partition : currentStateMap.keySet()) { Map stateMap = currentStateMap.get(partition); List preferenceList = new ArrayList<>(stateMap.keySet()); + + /** + * This sorting preserves the ordering of current state hosts in the order of current IS pref list + * Example: + * ideal state pref-list: [A, B, C] + * current-state: { + * A: FOLLOWER, + * B: LEADER, + * C: FOLLOWER + * } + * Lets say newPrefList = new ArrayList<>(current-state.keySet()) => [C, B, A] + * + * Sort 1: Sort based on preference-list order: + * -------------------------------------------------------- + * newPrefList = [C, B, A] => [A, B, C] + */ Collections.sort(preferenceList, new PreferenceListNodeComparator(stateMap, - clusterData.getStateModelDef(currentIdealState.getStateModelDefRef()), - Collections.emptyList())); + stateModelDef, currentIdealState.getPreferenceList(partition.getPartitionName()))); + + /** + * Sort 2: Sort based on state-priority order: + * -------------------------------------------------------- + * newPrefList = [A, B, C] => [B, A, C] + * Here, A will be 2nd and C will be third always as both have same priority so original (pref-list) order will be maintained. + */ + preferenceList.sort(new StatePriorityComparator(stateMap, stateModelDef)); + currentIdealState.setPreferenceList(partition.getPartitionName(), preferenceList); } LOG.info(String diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceComputationStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceComputationStage.java index 00b2fd71b1..e1d9a92152 100644 --- a/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceComputationStage.java +++ b/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceComputationStage.java @@ -91,7 +91,7 @@ private void processIdealStates(BaseControllerDataProvider cache, Map idealStates, boolean isTaskCache) { if (idealStates != null && idealStates.size() > 0) { for (IdealState idealState : idealStates.values()) { - if (idealState == null) { + if (idealState == null || idealState.getNumPartitions() == 0) { continue; } diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java index e074a22e68..06ea97fb9e 100644 --- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java +++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java @@ -788,6 +788,27 @@ public boolean remove(String path, int options) { return true; } + /** + * Sync remove with expected version. It tries to remove the ZNode if the ZNode's version matches + * the provided expectedVersion. This operation will FAIL if the node has any children. Node does + * not exist is regarded as success. If expectedVersion is set to -1, then the ZNode version + * match is not enforced. + */ + @Override + public boolean removeWithExpectedVersion(String path, int options, int expectedVersion) { + try { + // operation will not throw exception when path successfully deleted or does not exist + // despite real error, operation will throw exception when path not empty, and in this + // case, we do NOT try to delete recursively + _zkClient.delete(path, expectedVersion); + } catch (ZkException zkException) { + LOG.error("Failed to delete {} with opts {} and expected version {}.", path, options, + expectedVersion, zkException); + return false; + } + return true; + } + /** * async create. give up on error other than NONODE */ diff --git a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java index 6a635a5da7..2be1a4f769 100644 --- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java +++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java @@ -356,6 +356,31 @@ public boolean remove(String path, int options) { return _baseAccessor.remove(serverPath, options); } + @Override + public boolean removeWithExpectedVersion(String path, int options, int expectedVersion) { + String clientPath = path; + String serverPath = prependChroot(clientPath); + + Cache cache = getCache(serverPath); + if (cache != null) { + try { + cache.lockWrite(); + + boolean success = _baseAccessor.removeWithExpectedVersion(serverPath, options, expectedVersion); + if (success) { + cache.purgeRecursive(serverPath); + } + + return success; + } finally { + cache.unlockWrite(); + } + } + + // no cache + return _baseAccessor.removeWithExpectedVersion(serverPath, options, expectedVersion); + } + @Override public T get(String path, Stat stat, int options) { String clientPath = path; diff --git a/helix-core/src/main/java/org/apache/helix/store/zk/AutoFallbackPropertyStore.java b/helix-core/src/main/java/org/apache/helix/store/zk/AutoFallbackPropertyStore.java index c23bdca90f..ce82206a6e 100644 --- a/helix-core/src/main/java/org/apache/helix/store/zk/AutoFallbackPropertyStore.java +++ b/helix-core/src/main/java/org/apache/helix/store/zk/AutoFallbackPropertyStore.java @@ -95,6 +95,14 @@ public boolean remove(String path, int options) { return super.remove(path, options); } + @Override + public boolean removeWithExpectedVersion(String path, int options, int expectedVersion) { + if (_fallbackStore != null) { + _fallbackStore.removeWithExpectedVersion(path, options, expectedVersion); + } + return super.removeWithExpectedVersion(path, options, expectedVersion); + } + @Override public T get(String path, Stat stat, int options) { if (_fallbackStore == null) { diff --git a/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java new file mode 100644 index 0000000000..50cdd418f6 --- /dev/null +++ b/helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java @@ -0,0 +1,60 @@ +package org.apache.helix.controller.rebalancer; + +import java.util.List; +import java.util.Map; + +import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider; +import org.apache.helix.controller.stages.CurrentStateOutput; +import org.apache.helix.model.IdealState; +import org.apache.helix.model.MasterSlaveSMD; +import org.apache.helix.model.Partition; +import org.apache.helix.util.TestInputLoader; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestMaintenanceRebalancer { + + private static final String RESOURCE_NAME = "testResource"; + private static final String PARTITION_NAME = "testResourcePartition"; + + @Test(dataProvider = "TestComputeIdealStateInput") + public void testComputeIdealState(String comment, String stateModelName, List liveInstances, + List preferenceList, Map currentStateMap, List expectedPrefList) { + System.out.println("Test case comment: " + comment); + MaintenanceRebalancer rebalancer = new MaintenanceRebalancer(); + + Partition partition = new Partition(PARTITION_NAME); + CurrentStateOutput currentStateOutput = new CurrentStateOutput(); + for (String instance : currentStateMap.keySet()) { + currentStateOutput.setCurrentState(RESOURCE_NAME, partition, instance, currentStateMap.get(instance)); + } + + IdealState currentIdealState = new IdealState(RESOURCE_NAME); + currentIdealState.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO); + currentIdealState.setRebalancerClassName("org.apache.helix.controller.rebalancer.waged.WagedRebalancer"); + currentIdealState.setStateModelDefRef(stateModelName); + currentIdealState.setPreferenceList(PARTITION_NAME, preferenceList); + + ResourceControllerDataProvider dataCache = mock(ResourceControllerDataProvider.class); + when(dataCache.getStateModelDef("MasterSlave")).thenReturn(MasterSlaveSMD.build()); + + IdealState updatedIdealState = rebalancer + .computeNewIdealState(RESOURCE_NAME, currentIdealState, currentStateOutput, dataCache); + + List partitionPrefList = updatedIdealState.getPreferenceList(PARTITION_NAME); + Assert.assertTrue(partitionPrefList.equals(expectedPrefList)); + } + + @DataProvider(name = "TestComputeIdealStateInput") + public Object[][] loadTestComputeIdealStateInput() { + final String[] params = { + "comment", "stateModel", "liveInstances", "preferenceList", "currentStateMap", "expectedPreferenceList" + }; + return TestInputLoader.loadTestInputs("MaintenanceRebalancer.ComputeNewIdealState.json", params); + } + +} diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedClusterExpansionWithAddingResourcesBeforeInstances.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedClusterExpansionWithAddingResourcesBeforeInstances.java new file mode 100644 index 0000000000..de7aff6932 --- /dev/null +++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedClusterExpansionWithAddingResourcesBeforeInstances.java @@ -0,0 +1,189 @@ +package org.apache.helix.integration.rebalancer.WagedRebalancer; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; + +import org.apache.helix.ConfigAccessor; +import org.apache.helix.TestHelper; +import org.apache.helix.common.ZkTestBase; +import org.apache.helix.integration.manager.ClusterControllerManager; +import org.apache.helix.integration.manager.MockParticipantManager; +import org.apache.helix.model.ClusterConfig; +import org.apache.helix.model.ExternalView; +import org.apache.helix.model.IdealState; +import org.apache.helix.model.InstanceConfig; +import org.apache.helix.monitoring.mbeans.MonitorDomainNames; +import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier; +import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier; +import org.apache.helix.tools.ClusterVerifiers.StrictMatchExternalViewVerifier; +import org.apache.helix.tools.ClusterVerifiers.ZkHelixClusterVerifier; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.apache.helix.model.BuiltInStateModelDefinitions.LeaderStandby; +import static org.apache.helix.monitoring.mbeans.ClusterStatusMonitor.CLUSTER_DN_KEY; + +public class TestWagedClusterExpansionWithAddingResourcesBeforeInstances extends ZkTestBase { + private static final long TIMEOUT = 10 * 1000L; + protected static final AtomicLong PORT_GENERATOR = new AtomicLong(12918); + protected static final int PARTITIONS = 4; + + protected final String CLASS_NAME = getShortClassName(); + protected final String CLUSTER_NAME = CLUSTER_PREFIX + "_" + CLASS_NAME; + protected ClusterControllerManager _controller; + protected HelixClusterVerifier _clusterVerifier; + + List _participants = new ArrayList<>(); + Set _allDBs = new HashSet<>(); + int _replica = 3; + + @BeforeClass + public void setupCluster() throws Exception { + _gSetupTool.addCluster(CLUSTER_NAME, true); + + ConfigAccessor configAccessor = new ConfigAccessor(_gZkClient); + ClusterConfig clusterConfig = configAccessor.getClusterConfig(CLUSTER_NAME); + clusterConfig.setTopology("/zone/instance"); + clusterConfig.setFaultZoneType("zone"); + clusterConfig.setDelayRebalaceEnabled(true); + // Set a long enough time to ensure delayed rebalance is activate + clusterConfig.setRebalanceDelayTime(3000000); + + Map preference = new HashMap<>(); + preference.put(ClusterConfig.GlobalRebalancePreferenceKey.EVENNESS, 0); + preference.put(ClusterConfig.GlobalRebalancePreferenceKey.LESS_MOVEMENT, 10); + clusterConfig.setGlobalRebalancePreference(preference); + configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig); + + // create resource with instances + String testResource1 = "Test-resource-1"; + createResource(testResource1, 4, 4, "Tag-1", true); + + // start controller + String controllerName = CONTROLLER_PREFIX + "_0"; + _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName); + _controller.syncStart(); + + enablePersistBestPossibleAssignment(_gZkClient, CLUSTER_NAME, true); + enableTopologyAwareRebalance(_gZkClient, CLUSTER_NAME, true); + _gSetupTool.rebalanceResource(CLUSTER_NAME, testResource1, _replica); + _allDBs.add(testResource1); + + _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR) + .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME).setResources(_allDBs) + .build(); + Assert.assertTrue(_clusterVerifier.verify(12000)); + } + + private List createResource( + String resourceName, int numInstances, int numPartitions, String tagName, boolean enableParticipants) { + List nodes = new ArrayList<>(); + for (int i = 0; i < numInstances; i++) { + nodes.add(addInstance(new ConfigAccessor(_gZkClient), "zone-" + i % numInstances, tagName, enableParticipants)); + } + + createResourceWithWagedRebalance(CLUSTER_NAME, resourceName, LeaderStandby.name(), numPartitions, _replica, _replica - 1); + IdealState idealState = _gSetupTool.getClusterManagementTool().getResourceIdealState(CLUSTER_NAME, resourceName); + idealState.setInstanceGroupTag(tagName); + _gSetupTool.getClusterManagementTool().setResourceIdealState(CLUSTER_NAME, resourceName, idealState); + return nodes; + } + + private String addInstance(ConfigAccessor configAccessor, String zone, String instanceTag, boolean enabled) { + String storageNodeName = PARTICIPANT_PREFIX + "_" + PORT_GENERATOR.incrementAndGet(); + _gSetupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName); + _gSetupTool.addInstanceTag(CLUSTER_NAME, storageNodeName, instanceTag); + String domain = String.format("zone=%s,instance=%s", zone, storageNodeName); + + InstanceConfig instanceConfig = configAccessor.getInstanceConfig(CLUSTER_NAME, storageNodeName); + instanceConfig.setDomain(domain); + instanceConfig.setInstanceEnabled(enabled); + _gSetupTool.getClusterManagementTool().setInstanceConfig(CLUSTER_NAME, storageNodeName, instanceConfig); + + MockParticipantManager participant = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, storageNodeName); + if (enabled) { + // start dummy participant + participant.syncStart(); + } + _participants.add(participant); + + return storageNodeName; + } + + @AfterClass + public void afterClass() throws Exception { + _controller.syncStop(); + for (MockParticipantManager p : _participants) { + p.syncStop(); + } + deleteCluster(CLUSTER_NAME); + } + + @Test + public void testExpandClusterWithResourceWithoutInstances() throws Exception { + // Set-up a WAGED resource without any instances and let cluster rebalance successfully. + String testResource2 = "Test-resource-2"; + String testResourceTagName = "Tag-2"; + createResource(testResource2, 0, 0, testResourceTagName, false); + + _gSetupTool.rebalanceResource(CLUSTER_NAME, testResource2, _replica); + _allDBs.add(testResource2); + + ZkHelixClusterVerifier _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME) + .setZkClient(_gZkClient) + .setResources(_allDBs) + .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME) + .build(); + Assert.assertTrue(_clusterVerifier.verifyByPolling()); + checkRebalanceFailureGauge(false); + } + + private void checkRebalanceFailureGauge(final boolean expectFailure) throws Exception { + boolean result = TestHelper.verify(() -> { + try { + Long value = + (Long) _server.getAttribute(getMbeanName(CLUSTER_NAME), "RebalanceFailureGauge"); + return value != null && (value == 1) == expectFailure; + } catch (Exception e) { + return false; + } + }, TIMEOUT); + Assert.assertTrue(result); + } + + private ObjectName getMbeanName(String clusterName) throws MalformedObjectNameException { + String clusterBeanName = String.format("%s=%s", CLUSTER_DN_KEY, clusterName); + return new ObjectName( + String.format("%s:%s", MonitorDomainNames.ClusterStatus.name(), clusterBeanName)); + } + +} diff --git a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkBaseDataAccessor.java b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkBaseDataAccessor.java index bf4f342ddb..56337f5477 100644 --- a/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkBaseDataAccessor.java +++ b/helix-core/src/test/java/org/apache/helix/manager/zk/TestZkBaseDataAccessor.java @@ -543,6 +543,43 @@ public void testSyncRemove() { System.out.println("END " + testName + " at " + new Date(System.currentTimeMillis())); } + /** + * Test that remove with expected version will fail on version mismatch. Succeed on version match. + */ + @Test + public void testRemoveWithExpectedVersion() { + String className = TestHelper.getTestClassName(); + String methodName = TestHelper.getTestMethodName(); + String testName = className + "_" + methodName; + + System.out.println("START " + testName + " at " + new Date(System.currentTimeMillis())); + + String path = String.format("/%s/%s", _rootPath, "msg_0"); + ZNRecord record = new ZNRecord("msg_0"); + ZkBaseDataAccessor accessor = new ZkBaseDataAccessor(_gZkClient); + + // Create node + Assert.assertTrue(accessor.create(path, record, AccessOption.PERSISTENT)); + + // Create child node + String childPath = path + "/child"; + Assert.assertTrue(accessor.create(childPath, record, AccessOption.PERSISTENT)); + + // Delete parent with correct expected version. Should fail due to having a child + int currentVersion = accessor.getStat(path, 0).getVersion(); + Assert.assertFalse(accessor.removeWithExpectedVersion(path, 0, currentVersion)); + + // Remove Child + Assert.assertTrue(accessor.removeWithExpectedVersion(childPath, 0, -1)); + + // Delete childless node with wrong expected version. Should fail due to version mismatch + Assert.assertFalse(accessor.removeWithExpectedVersion(path, 0, currentVersion+100)); + + // Delete childless node with correct expected version. Shoudl succeed + Assert.assertTrue(accessor.removeWithExpectedVersion(path, 0, currentVersion)); + + } + @Test public void testDeleteNodeWithChildren() { String root = _rootPath; diff --git a/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java b/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java index 1567b98cc8..da5418d007 100644 --- a/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java +++ b/helix-core/src/test/java/org/apache/helix/mock/MockBaseDataAccessor.java @@ -111,6 +111,24 @@ public boolean remove(String path, int options) { return true; } + @Override + public boolean removeWithExpectedVersion(String path, int options, int expectedVersion) { + if (expectedVersion != -1) { + ZNode node = _recordMap.get(path); + if (node != null && node.getStat().getVersion() != expectedVersion) { + return false; + } + } + + _recordMap.remove(path); + try { + Thread.sleep(50); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return true; + } + @Override public boolean[] createChildren(List paths, List records, int options) { diff --git a/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json b/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json new file mode 100644 index 0000000000..9212b90bdd --- /dev/null +++ b/helix-core/src/test/resources/MaintenanceRebalancer.ComputeNewIdealState.json @@ -0,0 +1,98 @@ +[ + { + "comment": "[Case 1] Same state - No (A, B, C) -> (A, C, B) for different hash order", + "stateModel": "MasterSlave", + "liveInstances": [], + "preferenceList": [ + "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad" + ], + "currentStateMap": { + "lKmBBEPisM" : "MASTER", + "8PNpMU7EgT" : "SLAVE", + "HOs3rOFCad" : "SLAVE" + }, + "expectedPreferenceList": [ + "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad" + ] + }, + { + "comment": "[Case 2] Same state - No (A, B, C) -> (A, C, B) for different hash order", + "stateModel": "MasterSlave", + "liveInstances": [], + "preferenceList": [ + "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC" + ], + "currentStateMap": { + "Q3ZZuCeBpu" : "MASTER", + "QY9U91XBYo" : "SLAVE", + "GMYBW20gmC" : "SLAVE" + }, + "expectedPreferenceList": [ + "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC" + ] + }, + { + "comment": "[Case 3] second becomes leader, (A, B, C) -> (B, A, C) for different hash order", + "stateModel": "MasterSlave", + "liveInstances": [], + "preferenceList": [ + "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad" + ], + "currentStateMap": { + "lKmBBEPisM" : "SLAVE", + "8PNpMU7EgT" : "MASTER", + "HOs3rOFCad" : "SLAVE" + }, + "expectedPreferenceList": [ + "8PNpMU7EgT", "lKmBBEPisM", "HOs3rOFCad" + ] + }, + { + "comment": "[Case 4] second becomes leader, (A, B, C) -> (B, A, C) for different hash order", + "stateModel": "MasterSlave", + "liveInstances": [], + "preferenceList": [ + "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC" + ], + "currentStateMap": { + "Q3ZZuCeBpu" : "SLAVE", + "QY9U91XBYo" : "MASTER", + "GMYBW20gmC" : "SLAVE" + }, + "expectedPreferenceList": [ + "QY9U91XBYo", "Q3ZZuCeBpu", "GMYBW20gmC" + ] + }, + { + "comment": "[Case 5] leader becomes offline, (A, B, C) -> (B, C, A) for different hash order", + "stateModel": "MasterSlave", + "liveInstances": [], + "preferenceList": [ + "lKmBBEPisM", "8PNpMU7EgT", "HOs3rOFCad" + ], + "currentStateMap": { + "lKmBBEPisM" : "OFFLINE", + "8PNpMU7EgT" : "MASTER", + "HOs3rOFCad" : "SLAVE" + }, + "expectedPreferenceList": [ + "8PNpMU7EgT", "HOs3rOFCad", "lKmBBEPisM" + ] + }, + { + "comment": "[Case 6] leader becomes offline, (A, B, C) -> (B, C, A) for different hash order", + "stateModel": "MasterSlave", + "liveInstances": [], + "preferenceList": [ + "Q3ZZuCeBpu", "QY9U91XBYo", "GMYBW20gmC" + ], + "currentStateMap": { + "Q3ZZuCeBpu" : "OFFLINE", + "QY9U91XBYo" : "MASTER", + "GMYBW20gmC" : "SLAVE" + }, + "expectedPreferenceList": [ + "QY9U91XBYo", "GMYBW20gmC", "Q3ZZuCeBpu" + ] + } +] diff --git a/helix-front/package.json b/helix-front/package.json index e151010ae5..19f7efdb53 100644 --- a/helix-front/package.json +++ b/helix-front/package.json @@ -92,7 +92,7 @@ "core-js": "^2.4.1", "d3-shape": "^1.2.0", "dotenv": "^4.0.0", - "express": "^4.15.3", + "express": "^4.19.2", "express-session": "^1.15.6", "keycharm": "^0.3.0", "ldapjs": "^1.0.2", @@ -164,5 +164,9 @@ "typescript": "4.6.4", "util": "^0.12.4", "webpack": "5" + }, + "engines": { + "npm": ">=6.0.0 <7.0.0", + "node": ">=14.0.0 <15.0.0" } } diff --git a/helix-front/server/app.ts b/helix-front/server/app.ts index 3560ed010d..f346887111 100644 --- a/helix-front/server/app.ts +++ b/helix-front/server/app.ts @@ -18,6 +18,7 @@ import { } from './config'; import setRoutes from './routes'; +const isProd = process.env.NODE_ENV === 'production'; const httpsProxyAgent = PROXY_URL ? new ProxyAgent(PROXY_URL) : null; if (APP_INSIGHTS_CONNECTION_STRING) { @@ -35,7 +36,7 @@ if (APP_INSIGHTS_CONNECTION_STRING) { .start(); } -if (httpsProxyAgent && process.env.NODE_ENV === 'production') { +if (httpsProxyAgent && isProd) { // NOTES: // // - `defaultClient` property on `appInsights` doesn't exist @@ -52,13 +53,24 @@ const server = http.createServer(app); dotenv.load({ path: '.env' }); app.set('port', process.env.PORT || 4200); +const secretToken = process.env.SECRET_TOKEN; +if (!secretToken || secretToken === 'promiseyouwillchangeit') { + if (isProd) { + throw new Error('Please change your SECRET_TOKEN env'); + } else { + console.warn( + 'Remember to change your SECRET_TOKEN env before deploying to PROD' + ); + } +} + app.use('/', express.static(path.join(__dirname, '../public'))); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); app.use( session({ store: SESSION_STORE, - secret: process.env.SECRET_TOKEN, + secret: secretToken, resave: true, saveUninitialized: true, cookie: { expires: new Date(2147483647000) }, diff --git a/helix-front/yarn.lock b/helix-front/yarn.lock index 7e7fd22278..57efb41424 100644 --- a/helix-front/yarn.lock +++ b/helix-front/yarn.lock @@ -5070,13 +5070,13 @@ bluebird@^3.7.2: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -body-parser@1.20.1, body-parser@^1.17.2: - version "1.20.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" - integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== +body-parser@1.20.2, body-parser@^1.17.2: + version "1.20.2" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" + integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== dependencies: bytes "3.1.2" - content-type "~1.0.4" + content-type "~1.0.5" debug "2.6.9" depd "2.0.0" destroy "1.2.0" @@ -5084,7 +5084,7 @@ body-parser@1.20.1, body-parser@^1.17.2: iconv-lite "0.4.24" on-finished "2.4.1" qs "6.11.0" - raw-body "2.5.1" + raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" @@ -5884,10 +5884,10 @@ content-disposition@0.5.4: dependencies: safe-buffer "5.2.1" -content-type@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b" - integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA== +content-type@~1.0.4, content-type@~1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" + integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== continuation-local-storage@^3.2.1: version "3.2.1" @@ -5912,10 +5912,10 @@ cookie@0.4.2: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== -cookie@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" - integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.6.0.tgz#2798b04b071b0ecbff0dbb62a505a8efa4e19051" + integrity sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw== copy-anything@^2.0.1: version "2.0.6" @@ -7973,17 +7973,17 @@ express-session@^1.15.6: safe-buffer "5.2.1" uid-safe "~2.1.5" -express@^4.15.3, express@^4.17.1: - version "4.18.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" - integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== +express@^4.17.1, express@^4.19.2: + version "4.19.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" + integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.1" + body-parser "1.20.2" content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.5.0" + cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" @@ -8325,9 +8325,9 @@ flatted@^3.1.0: integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ== follow-redirects@^1.0.0: - version "1.15.4" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf" - integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw== + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== for-each@^0.3.3: version "0.3.3" @@ -13332,17 +13332,7 @@ range-parser@^1.2.1, range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.5.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" - integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== - dependencies: - bytes "3.1.2" - http-errors "2.0.0" - iconv-lite "0.4.24" - unpipe "1.0.0" - -raw-body@^2.2.0: +raw-body@2.5.2, raw-body@^2.2.0: version "2.5.2" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" integrity sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA== diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/api/client/RealmAwareZkClient.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/api/client/RealmAwareZkClient.java index ac0240d5f5..e24cc6e9cd 100644 --- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/api/client/RealmAwareZkClient.java +++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/api/client/RealmAwareZkClient.java @@ -254,6 +254,8 @@ String createEphemeralSequential(final String path, final Object data, final Lis boolean delete(final String path); + boolean delete(final String path, final int expectedVersion); + T readData(String path); T readData(String path, boolean returnNullIfPathNotExists); diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/DedicatedZkClient.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/DedicatedZkClient.java index 8dba7d81ef..a1a06f48da 100644 --- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/DedicatedZkClient.java +++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/DedicatedZkClient.java @@ -385,8 +385,13 @@ public void deleteRecursively(String path) { @Override public boolean delete(String path) { + return delete(path, -1); + } + + @Override + public boolean delete(String path, int expectedVersion) { checkIfPathContainsShardingKey(path); - return _rawZkClient.delete(path); + return _rawZkClient.delete(path, expectedVersion); } @Override diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/FederatedZkClient.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/FederatedZkClient.java index 8069985e3f..219ad9703c 100644 --- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/FederatedZkClient.java +++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/FederatedZkClient.java @@ -372,7 +372,12 @@ public void deleteRecursively(String path) { @Override public boolean delete(String path) { - return getZkClient(path).delete(path); + return delete(path, -1); + } + + @Override + public boolean delete(String path, int expectedVersion) { + return getZkClient(path).delete(path, expectedVersion); } @Override diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/SharedZkClient.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/SharedZkClient.java index 093538fa6b..3676288682 100644 --- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/SharedZkClient.java +++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/impl/client/SharedZkClient.java @@ -414,8 +414,13 @@ public void deleteRecursively(String path) { @Override public boolean delete(String path) { + return delete(path, -1); + } + + @Override + public boolean delete(String path, int expectedVersion) { checkIfPathContainsShardingKey(path); - return _innerSharedZkClient.delete(path); + return _innerSharedZkClient.delete(path, expectedVersion); } @Override diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/IZkConnection.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/IZkConnection.java index 43eefad26a..30dee94683 100644 --- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/IZkConnection.java +++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/IZkConnection.java @@ -45,6 +45,8 @@ public interface IZkConnection { public void delete(String path) throws InterruptedException, KeeperException; + public void delete (String path, int expectedVersion) throws InterruptedException, KeeperException; + boolean exists(final String path, final boolean watch) throws KeeperException, InterruptedException; List getChildren(final String path, final boolean watch) throws KeeperException, InterruptedException; diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java index 110587e09c..9d21e7604f 100644 --- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java +++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkClient.java @@ -2158,6 +2158,18 @@ public ZkLock getEventLock() { * @return true if path is successfully deleted, false if path does not exist */ public boolean delete(final String path) { + return delete(path, -1); + } + + /** + * Delete the given path. Path should not have any children or the deletion will fail. + * This function will throw an exception if we fail to delete an existing path. + * @param path Path of the znode to delete + * @param expectedVersion The expected version of the node to be removed, -1 means match any + * version. ZK + * @return true if the path is successfully deleted, false if path does not exist + */ + public boolean delete(final String path, final int expectedVersion) { long startT = System.currentTimeMillis(); boolean success; try { @@ -2166,7 +2178,7 @@ public boolean delete(final String path) { @Override public Object call() throws Exception { - getConnection().delete(path); + getConnection().delete(path, expectedVersion); return null; } }); diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkConnection.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkConnection.java index 376409231c..07d01df616 100644 --- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkConnection.java +++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/zkclient/ZkConnection.java @@ -145,6 +145,11 @@ public void delete(String path) throws InterruptedException, KeeperException { _zk.delete(path, -1); } + @Override + public void delete(String path, int expectedVersion) throws InterruptedException, KeeperException { + _zk.delete(path, expectedVersion); + } + @Override public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException { return _zk.exists(path, watch) != null; diff --git a/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/RealmAwareZkClientFactoryTestBase.java b/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/RealmAwareZkClientFactoryTestBase.java index 29c2df0f81..bae31de409 100644 --- a/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/RealmAwareZkClientFactoryTestBase.java +++ b/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/RealmAwareZkClientFactoryTestBase.java @@ -26,6 +26,7 @@ import org.apache.helix.zookeeper.api.client.RealmAwareZkClient; import org.apache.helix.zookeeper.datamodel.ZNRecord; import org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer; +import org.apache.helix.zookeeper.zkclient.exception.ZkBadVersionException; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -238,4 +239,27 @@ public void testDelete() { Assert.assertTrue(_realmAwareZkClient.delete(TEST_VALID_PATH)); Assert.assertFalse(_realmAwareZkClient.exists(TEST_VALID_PATH)); } + + /** + * Test that delete() with expected version fails on version mismatch. Succeeds on version match. + */ + @Test(dependsOnMethods = "testDelete") + public void testDeleteExpectedVersion() { + // Create a ZNode for testing + _realmAwareZkClient.createPersistent(TEST_VALID_PATH, true); + Assert.assertTrue(_realmAwareZkClient.exists(TEST_VALID_PATH)); + int expectedVersion = _realmAwareZkClient.getStat(TEST_VALID_PATH).getVersion(); + + // Test delete with invalid version, should fail to delete + try { + _realmAwareZkClient.delete(TEST_VALID_PATH, expectedVersion + 100); + Assert.fail("Should have thrown bad version exception"); + } catch (ZkBadVersionException expectedException) { + // Expected exception, continue + } + + // Assert delete with expected version successful + Assert.assertTrue(_realmAwareZkClient.delete(TEST_VALID_PATH, expectedVersion)); + Assert.assertFalse(_realmAwareZkClient.exists(TEST_VALID_PATH)); + } } diff --git a/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/TestFederatedZkClient.java b/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/TestFederatedZkClient.java index 30335efb13..3fea6a8bff 100644 --- a/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/TestFederatedZkClient.java +++ b/zookeeper-api/src/test/java/org/apache/helix/zookeeper/impl/client/TestFederatedZkClient.java @@ -40,6 +40,7 @@ import org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer; import org.apache.helix.zookeeper.routing.RoutingDataManager; import org.apache.helix.zookeeper.zkclient.IZkStateListener; +import org.apache.helix.zookeeper.zkclient.exception.ZkBadVersionException; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.Watcher; import org.testng.Assert; @@ -289,10 +290,33 @@ public void testDelete() { Assert.assertFalse(_realmAwareZkClient.exists(TEST_REALM_ONE_VALID_PATH)); } + /** + * Tests that delete() works on version match and fails on version mismatch + */ + @Test(dependsOnMethods = "testDelete") + public void testDeleteExpectedVersion() { + // Create a ZNode for testing + _realmAwareZkClient.createPersistent(TEST_VALID_PATH, true); + Assert.assertTrue(_realmAwareZkClient.exists(TEST_VALID_PATH)); + int expectedVersion = _realmAwareZkClient.getStat(TEST_VALID_PATH).getVersion(); + + // Test delete with invalid version, should fail to delete + try { + _realmAwareZkClient.delete(TEST_VALID_PATH, expectedVersion + 100); + Assert.fail("Should hvae thrown bad version exception"); + } catch (ZkBadVersionException expectedException) { + // Expected exception, continue + } + + // Assert delete with expected version successful + Assert.assertTrue(_realmAwareZkClient.delete(TEST_VALID_PATH, expectedVersion)); + Assert.assertFalse(_realmAwareZkClient.exists(TEST_VALID_PATH)); + } + /* * Tests that multi-realm feature. */ - @Test(dependsOnMethods = "testDelete") + @Test(dependsOnMethods = "testDeleteExpectedVersion") public void testMultiRealmCRUD() { ZNRecord realmOneZnRecord = new ZNRecord("realmOne"); realmOneZnRecord.setSimpleField("realmOne", "Value");