HDDS-13110. Add SCM metric for number of blocks deleted

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java

@@ -328,6 +328,7 @@ private Boolean checkInadequateReplica(Set<ContainerReplica> replicas,
   }
 
   private void addTxToTxSizeMap(DeletedBlocksTransaction tx) {
+    transactionStatusManager.recordTransactionBlockCount(tx.getTxID(), tx.getLocalIDCount());
     if (tx.hasTotalBlockReplicatedSize()) {
       transactionStatusManager.getTxSizeMap().put(tx.getTxID(),
           new SCMDeletedBlockTransactionStatusManager.TxBlockInfo(tx.getTxID(), tx.getContainerID(),
@@ -449,6 +450,8 @@ public DatanodeDeletedBlockTransactions getTransactions(
           getSCMDeletedBlockTransactionStatusManager().removeTransactionFromDNsCommitMap(txIDs);
           getSCMDeletedBlockTransactionStatusManager().removeTransactionFromDNsRetryCountMap(txIDs);
           metrics.incrBlockDeletionTransactionCompleted(txIDs.size());
+          metrics.incrNumBlocksDeleted(
+              transactionStatusManager.removeCompletedTransactionBlockCount(txIDs));
         }
       }
       return transactions;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java

@@ -73,6 +73,11 @@ public class SCMDeletedBlockTransactionStatusManager {
   private final Map<Long, Integer> transactionToRetryCountMap;
   // an in memory map to cache the size of each transaction sending to DN.
   private Map<Long, TxBlockInfo> txSizeMap;
+  // Maps txId to the number of blocks (localIDs) in the transaction. Used to
+  // count the number of blocks deleted when a transaction is fully completed.
+  // Populated when a transaction is prepared for sending and cleared when the
+  // transaction is removed from the log.
+  private final Map<Long, Integer> transactionToBlockCountMap;
 
   // The access to DeletedBlocksTXTable is protected by
   // DeletedBlockLogStateManager.
@@ -119,6 +124,7 @@ public SCMDeletedBlockTransactionStatusManager(
     this.transactionToDNsCommitMap = new ConcurrentHashMap<>();
     this.transactionToRetryCountMap = new ConcurrentHashMap<>();
     this.txSizeMap = new ConcurrentHashMap<>();
+    this.transactionToBlockCountMap = new ConcurrentHashMap<>();
     this.scmDeleteBlocksCommandStatusManager =
         new SCMDeleteBlocksCommandStatusManager(metrics);
     this.initDataDistributionData();
@@ -424,6 +430,7 @@ public void onBecomeLeader() {
     scmDeleteBlocksCommandStatusManager.clear();
     transactionToDNsCommitMap.clear();
     txSizeMap.clear();
+    transactionToBlockCountMap.clear();
     try {
       initDataDistributionData();
     } catch (IOException e) {
@@ -600,12 +607,37 @@ public void commitTransactions(List<DeleteBlockTransactionResult> transactionRes
     try {
       removeTransactions(txIDsToBeDeleted);
       metrics.incrBlockDeletionTransactionCompleted(txIDsToBeDeleted.size());
+      metrics.incrNumBlocksDeleted(removeCompletedTransactionBlockCount(txIDsToBeDeleted));
     } catch (IOException e) {
       LOG.warn("Could not commit delete block transactions: "
           + txIDsToBeDeleted, e);
     }
   }
 
+  /**
+   * Records the number of blocks in a transaction so that it can be counted
+   * towards the deleted blocks metric once the transaction is fully committed
+   * by all replicas and removed from the log.
+   */
+  void recordTransactionBlockCount(long txID, int blockCount) {
+    transactionToBlockCountMap.put(txID, blockCount);
+  }
+
+  /**
+   * Removes the given completed transactions from the block count map and
+   * returns the total number of blocks they contained.
+   */
+  long removeCompletedTransactionBlockCount(List<Long> txIDs) {
+    long blocks = 0;
+    for (Long txID : txIDs) {
+      Integer count = transactionToBlockCountMap.remove(txID);
+      if (count != null) {
+        blocks += count;
+      }
+    }
+    return blocks;
+  }
+
   public DeletedBlocksTransactionSummary getSummary() {
     return DeletedBlocksTransactionSummary.newBuilder()
         .setTotalTransactionCount(totalTxCount.get())

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/ScmBlockDeletingServiceMetrics.java

@@ -88,6 +88,10 @@ public final class ScmBlockDeletingServiceMetrics implements MetricsSource {
   @Metric(about = "The number of created txs which are added into DB.")
   private MutableCounterLong numBlockDeletionTransactionCreated;
 
+  @Metric(about = "The number of blocks deleted, i.e. the total number of blocks in all completed delete "
+      + "transactions that are fully committed by all replicas and removed from DB.")
+  private MutableCounterLong numBlocksDeleted;
+
   @Metric(about = "The number of skipped transactions")
   private MutableCounterLong numSkippedTransactions;
 
@@ -174,6 +178,10 @@ public void incrBlockDeletionTransactionCreated(long count) {
     this.numBlockDeletionTransactionCreated.incr(count);
   }
 
+  public void incrNumBlocksDeleted(long count) {
+    this.numBlocksDeleted.incr(count);
+  }
+
   public void incrSkippedTransaction() {
     this.numSkippedTransactions.incr();
   }
@@ -247,6 +255,14 @@ public long getNumBlockDeletionTransactionCreated() {
     return numBlockDeletionTransactionCreated.value();
   }
 
+  public long getNumBlocksDeleted() {
+    return numBlocksDeleted.value();
+  }
+
+  public long getNumBlockAddedForDeletionToDN() {
+    return numBlockAddedForDeletionToDN.value();
+  }
+
   public long getNumSkippedTransactions() {
     return numSkippedTransactions.value();
   }
@@ -270,6 +286,7 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) {
     numBlockDeletionTransactionFailureOnDatanodes.snapshot(builder, all);
     numBlockDeletionTransactionCompleted.snapshot(builder, all);
     numBlockDeletionTransactionCreated.snapshot(builder, all);
+    numBlocksDeleted.snapshot(builder, all);
     numSkippedTransactions.snapshot(builder, all);
     numProcessedTransactions.snapshot(builder, all);
     numBlockDeletionTransactionDataNodes.snapshot(builder, all);
@@ -420,6 +437,7 @@ public String toString() {
         .append("numBlockDeletionTransactionCreated = ").append(numBlockDeletionTransactionCreated.value()).append('\t')
         .append("numBlockDeletionTransactionCompleted = ")
         .append(numBlockDeletionTransactionCompleted.value()).append('\t')
+        .append("numBlocksDeleted = ").append(numBlocksDeleted.value()).append('\t')
         .append("numBlockDeletionCommandSent = ").append(numBlockDeletionCommandSent.value()).append('\t')
         .append("numBlockDeletionCommandSuccess = ").append(numBlockDeletionCommandSuccess.value()).append('\t')
         .append("numBlockDeletionCommandFailure = ").append(numBlockDeletionCommandFailure.value()).append('\t')

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java

@@ -316,6 +316,14 @@ public void testBlockDeletion(ReplicationConfig repConfig) throws Exception {
 
     assertEquals(metrics.getNumBlockDeletionTransactionCreated(),
         metrics.getNumBlockDeletionTransactionCompleted());
+    // Blocks are actually deleted: each completed transaction contributes at
+    // least one block, and the number of blocks deleted cannot exceed the
+    // number of blocks sent to datanodes for deletion.
+    assertThat(metrics.getNumBlocksDeleted()).isGreaterThan(0);
+    assertThat(metrics.getNumBlocksDeleted())
+        .isGreaterThanOrEqualTo(metrics.getNumBlockDeletionTransactionCompleted());
+    assertThat(metrics.getNumBlocksDeleted())
+        .isLessThanOrEqualTo(metrics.getNumBlockAddedForDeletionToDN());
     assertEquals(metrics.getNumBlockDeletionCommandSent(), metrics.getNumCommandsDatanodeSent());
     assertEquals(metrics.getNumBlockDeletionCommandSuccess(), metrics.getNumCommandsDatanodeSuccess());
     assertEquals(metrics.getBNumBlockDeletionCommandFailure(), metrics.getNumCommandsDatanodeFailed());