HDDS-15889. Show failed MoveResult breakdown in ozone admin containerbalancer status -v

hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto

@@ -675,6 +675,19 @@ message ContainerBalancerTaskIterationStatusInfoProto {
   repeated NodeTransferInfoProto sizeEnteringNodes = 9;
   repeated NodeTransferInfoProto sizeLeavingNodes = 10;
   optional int64 iterationDuration = 11;
+  repeated ContainerMoveFailureDetailProto containerMoveFailures = 12;
+}
+
+message ContainerMoveFailureDetailProto {
+  optional string reason = 1;
+  optional int64 count = 2;
+  repeated NodeFailureCountProto sourceFailureCounts = 3;
+  repeated NodeFailureCountProto targetFailureCounts = 4;
+}
+
+message NodeFailureCountProto {
+  optional string datanodeUuid = 1;
+  optional int64 count = 2;
 }
 
 message NodeTransferInfoProto {

hadoop-hdds/interface-admin/src/main/resources/proto.lock

@@ -2339,6 +2339,58 @@
                 "name": "iterationDuration",
                 "type": "int64",
                 "optional": true
+              },
+              {
+                "id": 12,
+                "name": "containerMoveFailures",
+                "type": "ContainerMoveFailureDetailProto",
+                "is_repeated": true
+              }
+            ]
+          },
+          {
+            "name": "ContainerMoveFailureDetailProto",
+            "fields": [
+              {
+                "id": 1,
+                "name": "reason",
+                "type": "string",
+                "optional": true
+              },
+              {
+                "id": 2,
+                "name": "count",
+                "type": "int64",
+                "optional": true
+              },
+              {
+                "id": 3,
+                "name": "sourceFailureCounts",
+                "type": "NodeFailureCountProto",
+                "is_repeated": true
+              },
+              {
+                "id": 4,
+                "name": "targetFailureCounts",
+                "type": "NodeFailureCountProto",
+                "is_repeated": true
+              }
+            ]
+          },
+          {
+            "name": "NodeFailureCountProto",
+            "fields": [
+              {
+                "id": 1,
+                "name": "datanodeUuid",
+                "type": "string",
+                "optional": true
+              },
+              {
+                "id": 2,
+                "name": "count",
+                "type": "int64",
+                "optional": true
               }
             ]
           },

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTask.java

@@ -37,6 +37,7 @@
 import java.util.Objects;
 import java.util.Queue;
 import java.util.Set;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -94,6 +95,7 @@ public class ContainerBalancerTask implements Runnable {
   private Set<String> includeNodes;
   private ContainerBalancerConfiguration config;
   private ContainerBalancerMetrics metrics;
+  private final ContainerMoveFailureTracker moveFailureTracker;
   private double upperLimit;
   private double lowerLimit;
   private ContainerBalancerSelectionCriteria selectionCriteria;
@@ -151,6 +153,7 @@ public ContainerBalancerTask(StorageContainerManager scm,
     this.containerBalancer = containerBalancer;
     this.config = config;
     this.metrics = metrics;
+    this.moveFailureTracker = new ContainerMoveFailureTracker();
     this.scmContext = scm.getScmContext();
     this.overUtilizedNodes = new ArrayList<>();
     this.underUtilizedNodes = new ArrayList<>();
@@ -346,7 +349,7 @@ private ContainerBalancerTaskIterationStatusInfo getIterationStatistic(Integer i
         currentIterationResultName,
         iterationDuration
     );
-    ContainerMoveInfo containerMoveInfo = new ContainerMoveInfo(metrics);
+    ContainerMoveInfo containerMoveInfo = new ContainerMoveInfo(metrics, moveFailureTracker);
 
     DataMoveInfo dataMoveInfo =
         getDataMoveInfo(sizeEnteringDataToNodes, sizeLeavingDataFromNodes);
@@ -793,7 +796,8 @@ private void checkIterationMoveResults() {
         LOG.warn("Container balancer is interrupted");
         Thread.currentThread().interrupt();
       } catch (TimeoutException e) {
-        long timeoutCounts = cancelMovesThatExceedTimeoutDuration();
+        long timeoutCounts = cancelMovesThatExceedTimeoutDuration(
+            ContainerMoveFailureReason.ITERATION_MOVE_TIMEOUT.name());
         LOG.warn("{} Container moves are canceled.", timeoutCounts);
         metrics.incrementNumContainerMovesTimeoutInLatestIteration(
             timeoutCounts);
@@ -836,7 +840,7 @@ private void checkIterationMoveResults() {
    * @return number of moves that did not complete (timed out) and were
    * cancelled.
    */
-  private long cancelMovesThatExceedTimeoutDuration() {
+  private long cancelMovesThatExceedTimeoutDuration(String reason) {
     Set<Map.Entry<ContainerMoveSelection,
         CompletableFuture<MoveManager.MoveResult>>>
         entries = moveSelectionToFutureMap.entrySet();
@@ -851,11 +855,13 @@ private long cancelMovesThatExceedTimeoutDuration() {
           CompletableFuture<MoveManager.MoveResult>>
           entry = iterator.next();
       if (!entry.getValue().isDone()) {
+        ContainerMoveSelection moveSelection = entry.getKey();
+        ContainerID containerID = moveSelection.getContainerID();
+        DatanodeDetails source = containerToSourceMap.get(containerID);
+        DatanodeDetails target = moveSelection.getTargetNode();
         LOG.warn("Container move timed out for container {} from source {}" +
-                " to target {}.", entry.getKey().getContainerID(),
-            containerToSourceMap.get(entry.getKey().getContainerID()),
-            entry.getKey().getTargetNode());
-
+                " to target {}.", containerID, source, target);
+        moveFailureTracker.recordFailure(reason, source, target);
         entry.getValue().cancel(true);
         numCancelled += 1;
       }
@@ -1004,11 +1010,15 @@ private boolean moveContainer(DatanodeDetails source,
         metrics.incrementCurrentIterationContainerMoveMetric(result, 1);
         moveSelectionToFutureMap.remove(moveSelection);
         if (ex != null) {
-          LOG.info("Container move for container {} from source {} to " +
-                  "target {} failed with exceptions.",
-              containerID, source,
-              moveSelection.getTargetNode(), ex);
-          metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+          if (!isCancellationCause(ex)) {
+            LOG.info("Container move for container {} from source {} to " +
+                    "target {} failed with exceptions.",
+                containerID, source,
+                moveSelection.getTargetNode(), ex);
+            metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+            moveFailureTracker.recordFailure(MoveManager.MoveResult.FAIL_UNEXPECTED_ERROR,
+                source, moveSelection.getTargetNode());
+          }
         } else {
           if (result == MoveManager.MoveResult.COMPLETED) {
             metrics.incrementDataSizeMovedInLatestIteration(containerInfo.getUsedBytes());
@@ -1021,6 +1031,7 @@ private boolean moveContainer(DatanodeDetails source,
                     " {} failed: {}",
                 moveSelection.getContainerID(), source,
                 moveSelection.getTargetNode(), result);
+            moveFailureTracker.recordFailure(result, source, moveSelection.getTargetNode());
           }
         }
       });
@@ -1032,14 +1043,20 @@ private boolean moveContainer(DatanodeDetails source,
       // exclude the container which caused failure of move to avoid error in next run.
       selectionCriteria.addToExcludeDueToFailContainers(moveSelection.getContainerID());
       metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+      moveFailureTracker.recordFailure(ContainerMoveFailureReason.PRE_MOVE_CONTAINER_NOT_FOUND.name(),
+          source, moveSelection.getTargetNode());
       return false;
     } catch (NodeNotFoundException e) {
       LOG.warn("Container move failed for container {}", containerID, e);
       metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+      moveFailureTracker.recordFailure(ContainerMoveFailureReason.PRE_MOVE_NODE_NOT_FOUND.name(),
+          source, moveSelection.getTargetNode());
       return false;
     } catch (ContainerReplicaNotFoundException e) {
       LOG.warn("Container move failed for container {}", containerID, e);
       metrics.incrementNumContainerMovesFailedInLatestIteration(1);
+      moveFailureTracker.recordFailure(ContainerMoveFailureReason.PRE_MOVE_REPLICA_NOT_FOUND.name(),
+          source, moveSelection.getTargetNode());
       // add source back to queue for replica not found only
       // the container is not excluded as it is a replica related failure
       findSourceStrategy.addBackSourceDataNode(source);
@@ -1225,6 +1242,11 @@ private void resetState() {
     metrics.resetDataSizeUnbalancedGB();
     metrics.resetNumDatanodesUnbalanced();
     metrics.resetNumContainerMovesFailedInLatestIteration();
+    moveFailureTracker.reset();
+  }
+
+  private static boolean isCancellationCause(Throwable ex) {
+    return ex instanceof CancellationException;
   }
 
   /**
@@ -1342,4 +1364,18 @@ enum Status {
     STOPPING,
     STOPPED
   }
+
+  /**
+   * Recorded when a scheduled container move fails before {@link MoveManager#move}
+   * completes, or when the balancer stops waiting for in-flight moves.
+   */
+  enum ContainerMoveFailureReason {
+    PRE_MOVE_CONTAINER_NOT_FOUND,
+    PRE_MOVE_NODE_NOT_FOUND,
+    PRE_MOVE_REPLICA_NOT_FOUND,
+    /**
+     * Move did not complete before the iteration move wait timeout expired.
+     */
+    ITERATION_MOVE_TIMEOUT
+  }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerTaskIterationStatusInfo.java

@@ -106,6 +106,14 @@ public long getContainerMovesTimeout() {
     return containerMoveInfo.getContainerMovesTimeout();
   }
 
+  /**
+   * Get per-reason failure summaries with per-datanode counts for this iteration.
+   * @return list of failure details, one entry per failure reason
+   */
+  public List<ContainerMoveFailureDetail> getFailures() {
+    return containerMoveInfo.getFailures();
+  }
+
   /**
    * Get a map of the node IDs and the corresponding data sizes moved to each node.
    * @return nodeId to size entering from node map
@@ -151,9 +159,35 @@ public StorageContainerLocationProtocolProtos.ContainerBalancerTaskIterationStat
         .addAllSizeLeavingNodes(
             mapToProtoNodeTransferInfo(getSizeLeavingNodes())
         )
+        .addAllContainerMoveFailures(mapToProtoFailures(getFailures()))
         .build();
   }
 
+  private List<StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto> mapToProtoFailures(
+      List<ContainerMoveFailureDetail> failures) {
+    return failures.stream()
+        .map(failure -> {
+          StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto.Builder builder =
+              StorageContainerLocationProtocolProtos.ContainerMoveFailureDetailProto.newBuilder()
+                  .setReason(failure.getReason())
+                  .setCount(failure.getCount());
+          failure.getSourceFailureCounts().forEach((uuid, count) ->
+              builder.addSourceFailureCounts(
+                  StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+                      .setDatanodeUuid(uuid)
+                      .setCount(count)
+                      .build()));
+          failure.getTargetFailureCounts().forEach((uuid, count) ->
+              builder.addTargetFailureCounts(
+                  StorageContainerLocationProtocolProtos.NodeFailureCountProto.newBuilder()
+                      .setDatanodeUuid(uuid)
+                      .setCount(count)
+                      .build()));
+          return builder.build();
+        })
+        .collect(Collectors.toList());
+  }
+
   /**
    * Converts an instance into the protobuf compatible object.
    * @param nodes node id to node traffic size

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveFailureDetail.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.container.balancer;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Per-reason container move failure summary with per-datanode failure counts.
+ */
+public final class ContainerMoveFailureDetail {
+  private final String reason;
+  private final long count;
+  private final Map<String, Long> sourceFailureCounts;
+  private final Map<String, Long> targetFailureCounts;
+
+  public ContainerMoveFailureDetail(String reason, long count,
+      Map<String, Long> sourceFailureCounts, Map<String, Long> targetFailureCounts) {
+    this.reason = reason;
+    this.count = count;
+    this.sourceFailureCounts = Collections.unmodifiableMap(sourceFailureCounts);
+    this.targetFailureCounts = Collections.unmodifiableMap(targetFailureCounts);
+  }
+
+  public String getReason() {
+    return reason;
+  }
+
+  public long getCount() {
+    return count;
+  }
+
+  public Map<String, Long> getSourceFailureCounts() {
+    return sourceFailureCounts;
+  }
+
+  public Map<String, Long> getTargetFailureCounts() {
+    return targetFailureCounts;
+  }
+}