New Release Snapshot

.github/workflows/helix-front.yml

@@ -1,9 +1,9 @@
 name: Helix Front CI
 on:
   pull_request:
-    branches: [ master ]
+    branches: [master]
     paths:
-      - 'helix-front/**'
+      - "helix-front/**"
 
 jobs:
   CI:
@@ -16,9 +16,9 @@ jobs:
       - name: Setup Node environment
         uses: actions/setup-node@v3
         with:
-          node-version: '16.x'
-          cache: 'yarn'
-          cache-dependency-path: 'helix-front/yarn.lock'
+          node-version: "14.x"
+          cache: "yarn"
+          cache-dependency-path: "helix-front/yarn.lock"
 
       - name: Install dependencies
         run: yarn

helix-core/pom.xml

@@ -179,14 +179,52 @@
       </resource>
     </resources>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.12.1</version>
+        <executions>
+          <execution>
+            <id>JDK 8</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.outputDirectory}_jdk8</outputDirectory>
+              <release>8</release>
+              <fork>true</fork>
+            </configuration>
+          </execution>
+          <execution>
+            <id>JDK 11</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <configuration>
+              <release>11</release>
+              <fork>true</fork>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
+        <version>3.3.0</version>
         <executions>
           <execution>
+            <id>default-package-jdk11</id>
+            <phase>package</phase>
             <goals>
+              <goal>jar</goal>
               <goal>test-jar</goal>
             </goals>
+            <configuration>
+              <classesDirectory>${project.build.outputDirectory}_jdk8</classesDirectory>
+              <classifier>jdk8</classifier>
+            </configuration>
           </execution>
         </executions>
       </plugin>

helix-core/src/main/java/org/apache/helix/BaseDataAccessor.java

@@ -112,6 +112,17 @@ default boolean multiSet(Map<String, DataUpdater<T>> updaterByPath) {
    */
   boolean remove(String path, int options);
 
+  /**
+   * This will remove the ZNode, if the ZNode's version matches the provided expectedVersion. This
+   * operation will fail if the node has any children.
+   * @param path Path to the ZNode to update
+   * @param options Set the type of ZNode see the valid values in {@link AccessOption}
+   * @param expectedVersion the expected version of the node to be removed, -1 means match any
+   * version
+   * @return true if the removal succeeded, false otherwise
+   */
+  boolean removeWithExpectedVersion(String path, int options, int expectedVersion);
+
   /**
    * Use it when creating children under a parent node. This will use async api for better
    * performance. If the child already exists it will return false.

helix-core/src/main/java/org/apache/helix/controller/rebalancer/MaintenanceRebalancer.java

@@ -28,6 +28,7 @@
 import org.apache.helix.controller.stages.CurrentStateOutput;
 import org.apache.helix.model.IdealState;
 import org.apache.helix.model.Partition;
+import org.apache.helix.model.StateModelDefinition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,12 +56,38 @@ public IdealState computeNewIdealState(String resourceName, IdealState currentId
 
     // One principal is to prohibit DROP -> OFFLINE and OFFLINE -> DROP state transitions.
     // Derived preference list from current state with state priority
+    StateModelDefinition stateModelDef = clusterData.getStateModelDef(currentIdealState.getStateModelDefRef());
+
     for (Partition partition : currentStateMap.keySet()) {
       Map<String, String> stateMap = currentStateMap.get(partition);
       List<String> preferenceList = new ArrayList<>(stateMap.keySet());
+
+      /**
+       * This sorting preserves the ordering of current state hosts in the order of current IS pref list
+       * Example:
+       * ideal state pref-list: [A, B, C]
+       * current-state: {
+       *     A: FOLLOWER,
+       *     B: LEADER,
+       *     C: FOLLOWER
+       * }
+       * Lets say newPrefList = new ArrayList<>(current-state.keySet()) => [C, B, A]
+       *
+       * Sort 1: Sort based on preference-list order:
+       * --------------------------------------------------------
+       * newPrefList = [C, B, A] => [A, B, C]
+       */
       Collections.sort(preferenceList, new PreferenceListNodeComparator(stateMap,
-          clusterData.getStateModelDef(currentIdealState.getStateModelDefRef()),
-          Collections.<String>emptyList()));
+          stateModelDef, currentIdealState.getPreferenceList(partition.getPartitionName())));
+
+      /**
+       * Sort 2: Sort based on state-priority order:
+       * --------------------------------------------------------
+       * newPrefList = [A, B, C] => [B, A, C]
+       * Here, A will be 2nd and C will be third always as both have same priority so original (pref-list) order will be maintained.
+       */
+      preferenceList.sort(new StatePriorityComparator(stateMap, stateModelDef));
+
       currentIdealState.setPreferenceList(partition.getPartitionName(), preferenceList);
     }
     LOG.info(String

helix-core/src/main/java/org/apache/helix/controller/stages/ResourceComputationStage.java

@@ -91,7 +91,7 @@ private void processIdealStates(BaseControllerDataProvider cache,
       Map<String, IdealState> idealStates, boolean isTaskCache) {
     if (idealStates != null && idealStates.size() > 0) {
       for (IdealState idealState : idealStates.values()) {
-        if (idealState == null) {
+        if (idealState == null || idealState.getNumPartitions() == 0) {
           continue;
         }
 

helix-core/src/main/java/org/apache/helix/manager/zk/ZkBaseDataAccessor.java

@@ -788,6 +788,27 @@ public boolean remove(String path, int options) {
     return true;
   }
 
+  /**
+   * Sync remove with expected version. It tries to remove the ZNode if the ZNode's version matches
+   * the provided expectedVersion. This operation will FAIL if the node has any children. Node does
+   * not exist is regarded as success. If expectedVersion is set to -1, then the ZNode version
+   * match is not enforced.
+   */
+  @Override
+  public boolean removeWithExpectedVersion(String path, int options, int expectedVersion) {
+    try {
+      // operation will not throw exception when path successfully deleted or does not exist
+      // despite real error, operation will throw exception when path not empty, and in this
+      // case, we do NOT try to delete recursively
+      _zkClient.delete(path, expectedVersion);
+    } catch (ZkException zkException) {
+      LOG.error("Failed to delete {} with opts {} and expected version {}.", path, options,
+          expectedVersion, zkException);
+      return false;
+    }
+    return true;
+  }
+
   /**
    * async create. give up on error other than NONODE
    */

helix-core/src/main/java/org/apache/helix/manager/zk/ZkCacheBaseDataAccessor.java

@@ -356,6 +356,31 @@ public boolean remove(String path, int options) {
     return _baseAccessor.remove(serverPath, options);
   }
 
+  @Override
+  public boolean removeWithExpectedVersion(String path, int options, int expectedVersion) {
+    String clientPath = path;
+    String serverPath = prependChroot(clientPath);
+
+    Cache<T> cache = getCache(serverPath);
+    if (cache != null) {
+      try {
+        cache.lockWrite();
+
+        boolean success = _baseAccessor.removeWithExpectedVersion(serverPath, options, expectedVersion);
+        if (success) {
+          cache.purgeRecursive(serverPath);
+        }
+
+        return success;
+      } finally {
+        cache.unlockWrite();
+      }
+    }
+
+    // no cache
+    return _baseAccessor.removeWithExpectedVersion(serverPath, options, expectedVersion);
+  }
+
   @Override
   public T get(String path, Stat stat, int options) {
     String clientPath = path;

helix-core/src/main/java/org/apache/helix/store/zk/AutoFallbackPropertyStore.java

@@ -95,6 +95,14 @@ public boolean remove(String path, int options) {
     return super.remove(path, options);
   }
 
+  @Override
+  public boolean removeWithExpectedVersion(String path, int options, int expectedVersion) {
+    if (_fallbackStore != null) {
+      _fallbackStore.removeWithExpectedVersion(path, options, expectedVersion);
+    }
+    return super.removeWithExpectedVersion(path, options, expectedVersion);
+  }
+
   @Override
   public T get(String path, Stat stat, int options) {
     if (_fallbackStore == null) {

helix-core/src/test/java/org/apache/helix/controller/rebalancer/TestMaintenanceRebalancer.java

@@ -0,0 +1,60 @@
+package org.apache.helix.controller.rebalancer;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.controller.stages.CurrentStateOutput;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.MasterSlaveSMD;
+import org.apache.helix.model.Partition;
+import org.apache.helix.util.TestInputLoader;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestMaintenanceRebalancer {
+
+  private static final String RESOURCE_NAME = "testResource";
+  private static final String PARTITION_NAME = "testResourcePartition";
+
+  @Test(dataProvider = "TestComputeIdealStateInput")
+  public void testComputeIdealState(String comment, String stateModelName, List<String> liveInstances,
+      List<String> preferenceList, Map<String, String> currentStateMap, List<String> expectedPrefList) {
+    System.out.println("Test case comment: " + comment);
+    MaintenanceRebalancer rebalancer = new MaintenanceRebalancer();
+
+    Partition partition = new Partition(PARTITION_NAME);
+    CurrentStateOutput currentStateOutput = new CurrentStateOutput();
+    for (String instance : currentStateMap.keySet()) {
+      currentStateOutput.setCurrentState(RESOURCE_NAME, partition, instance, currentStateMap.get(instance));
+    }
+
+    IdealState currentIdealState = new IdealState(RESOURCE_NAME);
+    currentIdealState.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO);
+    currentIdealState.setRebalancerClassName("org.apache.helix.controller.rebalancer.waged.WagedRebalancer");
+    currentIdealState.setStateModelDefRef(stateModelName);
+    currentIdealState.setPreferenceList(PARTITION_NAME, preferenceList);
+
+    ResourceControllerDataProvider dataCache = mock(ResourceControllerDataProvider.class);
+    when(dataCache.getStateModelDef("MasterSlave")).thenReturn(MasterSlaveSMD.build());
+
+    IdealState updatedIdealState = rebalancer
+        .computeNewIdealState(RESOURCE_NAME, currentIdealState, currentStateOutput, dataCache);
+
+    List<String> partitionPrefList = updatedIdealState.getPreferenceList(PARTITION_NAME);
+    Assert.assertTrue(partitionPrefList.equals(expectedPrefList));
+  }
+
+  @DataProvider(name = "TestComputeIdealStateInput")
+  public Object[][] loadTestComputeIdealStateInput() {
+    final String[] params = {
+        "comment", "stateModel", "liveInstances", "preferenceList", "currentStateMap", "expectedPreferenceList"
+    };
+    return TestInputLoader.loadTestInputs("MaintenanceRebalancer.ComputeNewIdealState.json", params);
+  }
+
+}