diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java index d9282dc0517d..9d03a6de2af8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java @@ -328,6 +328,7 @@ private Boolean checkInadequateReplica(Set replicas, } private void addTxToTxSizeMap(DeletedBlocksTransaction tx) { + transactionStatusManager.recordTransactionBlockCount(tx.getTxID(), tx.getLocalIDCount()); if (tx.hasTotalBlockReplicatedSize()) { transactionStatusManager.getTxSizeMap().put(tx.getTxID(), new SCMDeletedBlockTransactionStatusManager.TxBlockInfo(tx.getTxID(), tx.getContainerID(), @@ -449,6 +450,8 @@ public DatanodeDeletedBlockTransactions getTransactions( getSCMDeletedBlockTransactionStatusManager().removeTransactionFromDNsCommitMap(txIDs); getSCMDeletedBlockTransactionStatusManager().removeTransactionFromDNsRetryCountMap(txIDs); metrics.incrBlockDeletionTransactionCompleted(txIDs.size()); + metrics.incrNumBlocksDeleted( + transactionStatusManager.removeCompletedTransactionBlockCount(txIDs)); } } return transactions; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java index 43405ed90f74..617d67429381 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java @@ -73,6 +73,11 @@ public class SCMDeletedBlockTransactionStatusManager { private final Map transactionToRetryCountMap; // an in memory map to cache the size of each transaction sending to DN. private Map txSizeMap; + // Maps txId to the number of blocks (localIDs) in the transaction. Used to + // count the number of blocks deleted when a transaction is fully completed. + // Populated when a transaction is prepared for sending and cleared when the + // transaction is removed from the log. + private final Map transactionToBlockCountMap; // The access to DeletedBlocksTXTable is protected by // DeletedBlockLogStateManager. @@ -119,6 +124,7 @@ public SCMDeletedBlockTransactionStatusManager( this.transactionToDNsCommitMap = new ConcurrentHashMap<>(); this.transactionToRetryCountMap = new ConcurrentHashMap<>(); this.txSizeMap = new ConcurrentHashMap<>(); + this.transactionToBlockCountMap = new ConcurrentHashMap<>(); this.scmDeleteBlocksCommandStatusManager = new SCMDeleteBlocksCommandStatusManager(metrics); this.initDataDistributionData(); @@ -424,6 +430,7 @@ public void onBecomeLeader() { scmDeleteBlocksCommandStatusManager.clear(); transactionToDNsCommitMap.clear(); txSizeMap.clear(); + transactionToBlockCountMap.clear(); try { initDataDistributionData(); } catch (IOException e) { @@ -600,12 +607,37 @@ public void commitTransactions(List transactionRes try { removeTransactions(txIDsToBeDeleted); metrics.incrBlockDeletionTransactionCompleted(txIDsToBeDeleted.size()); + metrics.incrNumBlocksDeleted(removeCompletedTransactionBlockCount(txIDsToBeDeleted)); } catch (IOException e) { LOG.warn("Could not commit delete block transactions: " + txIDsToBeDeleted, e); } } + /** + * Records the number of blocks in a transaction so that it can be counted + * towards the deleted blocks metric once the transaction is fully committed + * by all replicas and removed from the log. + */ + void recordTransactionBlockCount(long txID, int blockCount) { + transactionToBlockCountMap.put(txID, blockCount); + } + + /** + * Removes the given completed transactions from the block count map and + * returns the total number of blocks they contained. + */ + long removeCompletedTransactionBlockCount(List txIDs) { + long blocks = 0; + for (Long txID : txIDs) { + Integer count = transactionToBlockCountMap.remove(txID); + if (count != null) { + blocks += count; + } + } + return blocks; + } + public DeletedBlocksTransactionSummary getSummary() { return DeletedBlocksTransactionSummary.newBuilder() .setTotalTransactionCount(totalTxCount.get()) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/ScmBlockDeletingServiceMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/ScmBlockDeletingServiceMetrics.java index 90be231d9ada..cd8e120bd69e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/ScmBlockDeletingServiceMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/ScmBlockDeletingServiceMetrics.java @@ -88,6 +88,10 @@ public final class ScmBlockDeletingServiceMetrics implements MetricsSource { @Metric(about = "The number of created txs which are added into DB.") private MutableCounterLong numBlockDeletionTransactionCreated; + @Metric(about = "The number of blocks deleted, i.e. the total number of blocks in all completed delete " + + "transactions that are fully committed by all replicas and removed from DB.") + private MutableCounterLong numBlocksDeleted; + @Metric(about = "The number of skipped transactions") private MutableCounterLong numSkippedTransactions; @@ -174,6 +178,10 @@ public void incrBlockDeletionTransactionCreated(long count) { this.numBlockDeletionTransactionCreated.incr(count); } + public void incrNumBlocksDeleted(long count) { + this.numBlocksDeleted.incr(count); + } + public void incrSkippedTransaction() { this.numSkippedTransactions.incr(); } @@ -247,6 +255,14 @@ public long getNumBlockDeletionTransactionCreated() { return numBlockDeletionTransactionCreated.value(); } + public long getNumBlocksDeleted() { + return numBlocksDeleted.value(); + } + + public long getNumBlockAddedForDeletionToDN() { + return numBlockAddedForDeletionToDN.value(); + } + public long getNumSkippedTransactions() { return numSkippedTransactions.value(); } @@ -270,6 +286,7 @@ public void getMetrics(MetricsCollector metricsCollector, boolean all) { numBlockDeletionTransactionFailureOnDatanodes.snapshot(builder, all); numBlockDeletionTransactionCompleted.snapshot(builder, all); numBlockDeletionTransactionCreated.snapshot(builder, all); + numBlocksDeleted.snapshot(builder, all); numSkippedTransactions.snapshot(builder, all); numProcessedTransactions.snapshot(builder, all); numBlockDeletionTransactionDataNodes.snapshot(builder, all); @@ -420,6 +437,7 @@ public String toString() { .append("numBlockDeletionTransactionCreated = ").append(numBlockDeletionTransactionCreated.value()).append('\t') .append("numBlockDeletionTransactionCompleted = ") .append(numBlockDeletionTransactionCompleted.value()).append('\t') + .append("numBlocksDeleted = ").append(numBlocksDeleted.value()).append('\t') .append("numBlockDeletionCommandSent = ").append(numBlockDeletionCommandSent.value()).append('\t') .append("numBlockDeletionCommandSuccess = ").append(numBlockDeletionCommandSuccess.value()).append('\t') .append("numBlockDeletionCommandFailure = ").append(numBlockDeletionCommandFailure.value()).append('\t') diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java index 3c561c93f9ea..d33b8602ad92 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java @@ -316,6 +316,14 @@ public void testBlockDeletion(ReplicationConfig repConfig) throws Exception { assertEquals(metrics.getNumBlockDeletionTransactionCreated(), metrics.getNumBlockDeletionTransactionCompleted()); + // Blocks are actually deleted: each completed transaction contributes at + // least one block, and the number of blocks deleted cannot exceed the + // number of blocks sent to datanodes for deletion. + assertThat(metrics.getNumBlocksDeleted()).isGreaterThan(0); + assertThat(metrics.getNumBlocksDeleted()) + .isGreaterThanOrEqualTo(metrics.getNumBlockDeletionTransactionCompleted()); + assertThat(metrics.getNumBlocksDeleted()) + .isLessThanOrEqualTo(metrics.getNumBlockAddedForDeletionToDN()); assertEquals(metrics.getNumBlockDeletionCommandSent(), metrics.getNumCommandsDatanodeSent()); assertEquals(metrics.getNumBlockDeletionCommandSuccess(), metrics.getNumCommandsDatanodeSuccess()); assertEquals(metrics.getBNumBlockDeletionCommandFailure(), metrics.getNumCommandsDatanodeFailed());