Skip to content

Commit

Permalink
HDDS-11367. Fix review notice. Refactor saving iteration statistic.
Browse files Browse the repository at this point in the history
  • Loading branch information
@juncevich
juncevich committed Nov 26, 2024
1 parent 15a8558 commit 2e60bf3
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 33 deletions.
61 changes: 39 additions & 22 deletions ...cm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,8 +274,10 @@ private void balance() {
return;
}

IterationResult iR = doIteration();
saveIterationStatistic(i + 1, iR);
IterationResult currentIterationResult = doIteration();
ContainerBalancerTaskIterationStatusInfo iterationStatistic =
getIterationStatistic(i + 1, currentIterationResult, getCurrentIterationDuration());
iterationsStatistic.offer(iterationStatistic);

isCurrentIterationInProgress.compareAndSet(true, false);

Expand All @@ -284,17 +286,17 @@ private void balance() {

metrics.incrementNumIterations(1);

LOG.info("Result of this iteration of Container Balancer: {}", iR);
LOG.info("Result of this iteration of Container Balancer: {}", currentIterationResult);

// if no new move option is generated, it means the cluster cannot be
// balanced anymore; so just stop balancer
if (iR == IterationResult.CAN_NOT_BALANCE_ANY_MORE) {
tryStopWithSaveConfiguration(iR.toString());
if (currentIterationResult == IterationResult.CAN_NOT_BALANCE_ANY_MORE) {
tryStopWithSaveConfiguration(currentIterationResult.toString());
return;
}

// persist next iteration index
if (iR == IterationResult.ITERATION_COMPLETED) {
if (currentIterationResult == IterationResult.ITERATION_COMPLETED) {
try {
saveConfiguration(config, true, i + 1);
} catch (IOException | TimeoutException e) {
Expand Down Expand Up @@ -325,30 +327,45 @@ private void balance() {
tryStopWithSaveConfiguration("Completed all iterations.");
}

private void saveIterationStatistic(Integer iterationNumber, IterationResult currentIterationResult) {
long iterationDuration = now().toEpochSecond() - currentIterationStarted.toEpochSecond();
private ContainerBalancerTaskIterationStatusInfo getIterationStatistic(Integer iterationNumber,
IterationResult currentIterationResult,
long iterationDuration) {
String currentIterationResultName = currentIterationResult == null ? null : currentIterationResult.name();
Map<UUID, Long> sizeEnteringDataToNodes =
convertToNodeIdToTrafficMap(findTargetStrategy.getSizeEnteringNodes());
Map<UUID, Long> sizeLeavingDataFromNodes =
convertToNodeIdToTrafficMap(findSourceStrategy.getSizeLeavingNodes());
IterationInfo iterationInfo = new IterationInfo(
iterationNumber,
currentIterationResult.name(),
currentIterationResultName,
iterationDuration
);
ContainerMoveInfo containerMoveInfo = new ContainerMoveInfo(metrics);
DataMoveInfo dataMoveInfo = new DataMoveInfo(
getSizeScheduledForMoveInLatestIteration(),
metrics.getDataSizeMovedInLatestIteration(),
sizeEnteringDataToNodes,
sizeLeavingDataFromNodes
);
ContainerBalancerTaskIterationStatusInfo iterationStatistic = new ContainerBalancerTaskIterationStatusInfo(
iterationInfo,
containerMoveInfo,
dataMoveInfo
);
iterationsStatistic.offer(iterationStatistic);

DataMoveInfo dataMoveInfo =
getDataMoveInfo(currentIterationResultName, sizeEnteringDataToNodes, sizeLeavingDataFromNodes);
return new ContainerBalancerTaskIterationStatusInfo(iterationInfo, containerMoveInfo, dataMoveInfo);
}

private DataMoveInfo getDataMoveInfo(String currentIterationResultName, Map<UUID, Long> sizeEnteringDataToNodes,
Map<UUID, Long> sizeLeavingDataFromNodes) {
if (currentIterationResultName == null) {
// For unfinished iteration
return new DataMoveInfo(
getSizeScheduledForMoveInLatestIteration(),
sizeActuallyMovedInLatestIteration,
sizeEnteringDataToNodes,
sizeLeavingDataFromNodes
);
} else {
// For finished iteration
return new DataMoveInfo(
getSizeScheduledForMoveInLatestIteration(),
metrics.getDataSizeMovedInLatestIteration(),
sizeEnteringDataToNodes,
sizeLeavingDataFromNodes
);
}
}

private Map<UUID, Long> convertToNodeIdToTrafficMap(Map<DatanodeDetails, Long> nodeTrafficMap) {
Expand Down Expand Up @@ -388,7 +405,7 @@ private ContainerBalancerTaskIterationStatusInfo createCurrentIterationStatistic
long iterationDuration = getCurrentIterationDuration();

if (isCurrentIterationInProgress.get()) {
return getFilledCurrentIterationStatistic(lastIterationNumber, iterationDuration);
return getIterationStatistic(lastIterationNumber + 1, null, iterationDuration);
} else {
return null;
}
Expand Down
2 changes: 1 addition & 1 deletion ...t/java/org/apache/hadoop/hdds/scm/container/balancer/TestContainerBalancerStatusInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ void testGetCurrentStatisticsWithDelay() throws Exception {
config.setMaxSizeToMovePerIteration(50 * OzoneConsts.GB);
OzoneConfiguration configuration = new OzoneConfiguration();
configuration.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "1");
ContainerBalancerTask task = mockedScm.startBalancerTaskAsync(config, configuration,true);
ContainerBalancerTask task = mockedScm.startBalancerTaskAsync(config, configuration, true);
// Delay in finishing the first iteration
LambdaTestUtils.await(1100, 50, () -> task.getCurrentIterationsStatistic().size() == 1);
List<ContainerBalancerTaskIterationStatusInfo> iterationsStatic = task.getCurrentIterationsStatistic();
Expand Down
18 changes: 14 additions & 4 deletions ...tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,25 @@ public void execute(ScmClient scmClient) throws IOException {
= balancerStatusInfo.getIterationsStatusInfoList();

System.out.println("Current iteration info:");
System.out.println(
getPrettyIterationStatusInfo(iterationsStatusInfoList.get(iterationsStatusInfoList.size() - 1))
);
ContainerBalancerTaskIterationStatusInfoProto currentIterationStatistic = iterationsStatusInfoList.stream()
.filter(it -> it.getIterationResult().isEmpty())
.findFirst()
.orElse(null);
if (currentIterationStatistic == null) {
System.out.println("-\n");
} else {
System.out.println(
getPrettyIterationStatusInfo(currentIterationStatistic)
);
}


if (verboseWithHistory) {
System.out.println("Iteration history list:");
System.out.println(
iterationsStatusInfoList.subList(0, iterationsStatusInfoList.size() - 1)
iterationsStatusInfoList
.stream()
.filter(it -> !it.getIterationResult().isEmpty())
.map(this::getPrettyIterationStatusInfo)
.collect(Collectors.joining("\n"))
);
Expand Down
12 changes: 6 additions & 6 deletions hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,12 @@ Verify Balancer Iteration
Verify Balancer Iteration History
[arguments] ${output}
Should Contain ${output} Iteration history list:
Should Contain X Times ${output} Size scheduled to move 2 collapse_spaces=True
Should Contain X Times ${output} Moved data size 2 collapse_spaces=True
Should Contain X Times ${output} Scheduled to move containers 2 collapse_spaces=True
Should Contain X Times ${output} Already moved containers 2 collapse_spaces=True
Should Contain X Times ${output} Failed to move containers 0 2 collapse_spaces=True
Should Contain X Times ${output} Failed to move containers by timeout 0 2 collapse_spaces=True
Should Contain X Times ${output} Size scheduled to move 1 collapse_spaces=True
Should Contain X Times ${output} Moved data size 1 collapse_spaces=True
Should Contain X Times ${output} Scheduled to move containers 1 collapse_spaces=True
Should Contain X Times ${output} Already moved containers 1 collapse_spaces=True
Should Contain X Times ${output} Failed to move containers 0 1 collapse_spaces=True
Should Contain X Times ${output} Failed to move containers by timeout 0 1 collapse_spaces=True
Should Contain ${output} Iteration result ITERATION_COMPLETED collapse_spaces=True

Run Balancer Status
Expand Down

0 comments on commit 2e60bf3

Please sign in to comment.