From 74ca8327f38bf8a5d7ac518b398db471e38e393f Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 9 Apr 2026 23:54:34 -0700 Subject: [PATCH 1/4] HDDS-14996. Add metrics to track Snapshot backup SST file stats. Change-Id: Ib531e9a99d2e98a2e7b2a8ec4e39a3c36cd9aaa0 --- .../snapshot/OMSnapshotDirectoryMetrics.java | 157 +++++++++++++----- .../TestOMSnapshotDirectoryMetrics.java | 133 +++++++++++++++ 2 files changed, 249 insertions(+), 41 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java index 9e485b769eb5..5ab945c1ca2e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java @@ -68,6 +68,8 @@ public final class OMSnapshotDirectoryMetrics extends OMPeriodicMetrics implemen private @Metric MutableGaugeLong dbSnapshotsDirSize; private @Metric MutableGaugeLong totalSstFilesCount; private @Metric MutableGaugeLong numSnapshots; + private @Metric MutableGaugeLong sstBackupDirSize; + private @Metric MutableGaugeLong sstBackupSstFilesCount; private final OMMetadataManager metadataManager; private final MetricsRegistry registry = new MetricsRegistry(SOURCE_NAME); @@ -79,6 +81,21 @@ public final class OMSnapshotDirectoryMetrics extends OMPeriodicMetrics implemen OZONE_OM_SNAPSHOT_DIRECTORY_METRICS_UPDATE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS)); this.metadataManager = metadataManager; + this.dbSnapshotsDirSize = registry.newGauge( + SnapshotMetricsInfo.DbSnapshotsDirSize.name(), + SnapshotMetricsInfo.DbSnapshotsDirSize.description(), 0L); + this.totalSstFilesCount = registry.newGauge( + SnapshotMetricsInfo.TotalSstFilesCount.name(), + SnapshotMetricsInfo.TotalSstFilesCount.description(), 0L); + this.numSnapshots = registry.newGauge( + SnapshotMetricsInfo.NumSnapshots.name(), + SnapshotMetricsInfo.NumSnapshots.description(), 0L); + this.sstBackupDirSize = registry.newGauge( + SnapshotMetricsInfo.SstBackupDirSize.name(), + SnapshotMetricsInfo.SstBackupDirSize.description(), 0L); + this.sstBackupSstFilesCount = registry.newGauge( + SnapshotMetricsInfo.SstBackupSstFilesCount.name(), + SnapshotMetricsInfo.SstBackupSstFilesCount.description(), 0L); } public static OMSnapshotDirectoryMetrics create(ConfigurationSource conf, @@ -116,7 +133,10 @@ protected boolean updateMetrics() { try { // Calculate aggregate metrics - calculateAndUpdateMetrics(snapshotsDir); + String sstBackupDir = store.getRocksDBCheckpointDiffer() != null + ? store.getRocksDBCheckpointDiffer().getSSTBackupDir() : null; + calculateAndUpdateMetrics(snapshotsDir, + sstBackupDir != null ? new File(sstBackupDir) : null); } catch (Exception e) { LOG.warn("Error calculating snapshot directory metrics", e); resetMetrics(); @@ -132,54 +152,93 @@ protected boolean updateMetrics() { * * @param directory the directory containing all checkpointDirs. */ - private void calculateAndUpdateMetrics(File directory) throws IOException { - Set visitedInodes = new HashSet<>(); - long totalSize = 0; - long sstFileCount = 0; + private void calculateAndUpdateMetrics(File snapshotsDir, + File sstBackupDir) throws IOException { + Set visitedSnapshotsInodes = new HashSet<>(); + long snapshotsTotalSize = 0; + long snapshotsSstFileCount = 0; int snapshotCount = 0; - try (Stream checkpointDirs = Files.list(directory.toPath())) { - for (Path checkpointDir : checkpointDirs.collect(Collectors.toList())) { - if (Files.isDirectory(checkpointDir)) { - snapshotCount++; - try (Stream files = Files.list(checkpointDir)) { - for (Path path : files.collect(Collectors.toList())) { - if (Files.isRegularFile(path)) { - try { - // Get inode number - Object fileKey = IOUtils.getINode(path); - if (fileKey == null) { - // Fallback: use file path + size as unique identifier - fileKey = path.toAbsolutePath() + ":" + Files.size(path); - } - // Only count this file if we haven't seen this inode before - if (visitedInodes.add(fileKey)) { - if (path.toFile().getName().endsWith(ROCKSDB_SST_SUFFIX)) { - sstFileCount++; - } - totalSize += Files.size(path); - } - } catch (UnsupportedOperationException | IOException e) { - // Fallback: if we can't get inode, just count the file size. - LOG.warn("Could not get inode for {}, using file size directly: {}", - path, e.getMessage()); - totalSize += Files.size(path); - if (path.toFile().getName().endsWith(ROCKSDB_SST_SUFFIX)) { - sstFileCount++; - } - } - } - } + + if (snapshotsDir != null && snapshotsDir.exists() && snapshotsDir.isDirectory()) { + try (Stream checkpointDirs = Files.list(snapshotsDir.toPath())) { + for (Path checkpointDir : checkpointDirs.collect(Collectors.toList())) { + if (Files.isDirectory(checkpointDir)) { + snapshotCount++; + SizeAndCount sizeAndCount = + calculateDirSize(checkpointDir, visitedSnapshotsInodes); + snapshotsTotalSize += sizeAndCount.size; + snapshotsSstFileCount += sizeAndCount.count; } } } } + + long backupDirSize = 0; + long backupSstFileCount = 0; + if (sstBackupDir != null && sstBackupDir.exists() && + sstBackupDir.isDirectory()) { + SizeAndCount sizeAndCount = + calculateDirSize(sstBackupDir.toPath(), new HashSet<>()); + backupDirSize = sizeAndCount.size; + backupSstFileCount = sizeAndCount.count; + } + numSnapshots.set(snapshotCount); - totalSstFilesCount.set(sstFileCount); - dbSnapshotsDirSize.set(totalSize); + totalSstFilesCount.set(snapshotsSstFileCount); + dbSnapshotsDirSize.set(snapshotsTotalSize); + sstBackupDirSize.set(backupDirSize); + sstBackupSstFilesCount.set(backupSstFileCount); if (LOG.isDebugEnabled()) { - LOG.debug("Updated snapshot directory metrics: size={}, sstFiles={}, snapshots={}", - totalSize, sstFileCount, snapshotCount); + LOG.debug("Updated snapshot directory metrics: size={}, sstFiles={}, " + + "snapshots={}, backupSize={}, backupSstFiles={}", + snapshotsTotalSize, snapshotsSstFileCount, snapshotCount, + backupDirSize, backupSstFileCount); + } + } + + private SizeAndCount calculateDirSize(Path directory, Set visitedInodes) throws IOException { + long size = 0; + long count = 0; + try (Stream files = Files.list(directory)) { + for (Path path : files.collect(Collectors.toList())) { + if (Files.isRegularFile(path)) { + try { + // Get inode number + Object fileKey = IOUtils.getINode(path); + if (fileKey == null) { + // Fallback: use file path + size as unique identifier + fileKey = path.toAbsolutePath() + ":" + Files.size(path); + } + // Only count this file if we haven't seen this inode before + if (visitedInodes.add(fileKey)) { + if (path.toFile().getName().endsWith(ROCKSDB_SST_SUFFIX)) { + count++; + } + size += Files.size(path); + } + } catch (UnsupportedOperationException | IOException e) { + // Fallback: if we can't get inode, just count the file size. + LOG.warn("Could not get inode for {}, using file size directly: {}", + path, e.getMessage()); + size += Files.size(path); + if (path.toFile().getName().endsWith(ROCKSDB_SST_SUFFIX)) { + count++; + } + } + } + } + } + return new SizeAndCount(size, count); + } + + private static final class SizeAndCount { + private final long size; + private final long count; + + private SizeAndCount(long size, long count) { + this.size = size; + this.count = count; } } @@ -190,6 +249,8 @@ private void resetMetrics() { dbSnapshotsDirSize.set(0); totalSstFilesCount.set(0); numSnapshots.set(0); + sstBackupDirSize.set(0); + sstBackupSstFilesCount.set(0); } /** @@ -204,6 +265,8 @@ public void getMetrics(MetricsCollector collector, boolean all) { .addGauge(SnapshotMetricsInfo.DbSnapshotsDirSize, dbSnapshotsDirSize.value()) .addGauge(SnapshotMetricsInfo.TotalSstFilesCount, totalSstFilesCount.value()) .addGauge(SnapshotMetricsInfo.NumSnapshots, numSnapshots.value()) + .addGauge(SnapshotMetricsInfo.SstBackupDirSize, sstBackupDirSize.value()) + .addGauge(SnapshotMetricsInfo.SstBackupSstFilesCount, sstBackupSstFilesCount.value()) .addGauge(SnapshotMetricsInfo.LastUpdateTime, getLastUpdateTime()); } @@ -222,6 +285,16 @@ public long getNumSnapshots() { return numSnapshots.value(); } + @VisibleForTesting + public long getSstBackupDirSize() { + return sstBackupDirSize.value(); + } + + @VisibleForTesting + public long getSstBackupSstFilesCount() { + return sstBackupSstFilesCount.value(); + } + public void unRegister() { stop(); MetricsSystem ms = DefaultMetricsSystem.instance(); @@ -236,6 +309,8 @@ enum SnapshotMetricsInfo implements MetricsInfo { DbSnapshotsDirSize("Total size of db.snapshots directory in bytes"), TotalSstFilesCount("Total number of SST files across all snapshots"), NumSnapshots("Total number of snapshot checkpoint directories"), + SstBackupDirSize("Total size of backup SST directory in bytes"), + SstBackupSstFilesCount("Total number of SST files in backup SST directory"), LastUpdateTime("Time stamp when the snapshot directory metrics were last updated"); private final String desc; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java new file mode 100644 index 000000000000..728909dd571f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +/** + * Tests for OMSnapshotDirectoryMetrics. + */ +public class TestOMSnapshotDirectoryMetrics { + + @TempDir + private Path tempDir; + + private OMMetadataManager metadataManager; + private RDBStore store; + private RocksDBCheckpointDiffer differ; + private OMSnapshotDirectoryMetrics metrics; + + @BeforeEach + public void setup() throws IOException { + metadataManager = Mockito.mock(OMMetadataManager.class); + store = Mockito.mock(RDBStore.class); + differ = Mockito.mock(RocksDBCheckpointDiffer.class); + + when(metadataManager.getStore()).thenReturn(store); + when(store.getRocksDBCheckpointDiffer()).thenReturn(differ); + + Path snapshotsDir = tempDir.resolve("snapshots"); + Files.createDirectories(snapshotsDir); + when(store.getSnapshotsParentDir()).thenReturn(snapshotsDir.toString()); + + Path backupDir = tempDir.resolve("backup"); + Files.createDirectories(backupDir); + when(differ.getSSTBackupDir()).thenReturn(backupDir.toString()); + + OzoneConfiguration conf = new OzoneConfiguration(); + metrics = new OMSnapshotDirectoryMetrics(conf, metadataManager); + } + + @Test + public void testMetrics() throws IOException { + Path snapshotsDir = Paths.get(metadataManager.getStore().getSnapshotsParentDir()); + Path backupDir = Paths.get(metadataManager.getStore().getRocksDBCheckpointDiffer().getSSTBackupDir()); + + // Create a snapshot directory with some files + Path snap1 = snapshotsDir.resolve(UUID.randomUUID().toString()); + Files.createDirectories(snap1); + Path file1 = snap1.resolve("file1" + ROCKSDB_SST_SUFFIX); + Files.write(file1, "data1".getBytes(StandardCharsets.UTF_8)); + long size1 = Files.size(file1); + + // Create backup directory with some files + Path backupFile1 = backupDir.resolve("backup1" + ROCKSDB_SST_SUFFIX); + Files.write(backupFile1, "backupData1".getBytes(StandardCharsets.UTF_8)); + long backupSize1 = Files.size(backupFile1); + + Path backupFile2 = backupDir.resolve("backup2" + ROCKSDB_SST_SUFFIX); + Files.write(backupFile2, "backupData22".getBytes(StandardCharsets.UTF_8)); + long backupSize2 = Files.size(backupFile2); + + metrics.updateMetrics(); + + assertEquals(1, metrics.getNumSnapshots()); + assertEquals(1, metrics.getTotalSstFilesCount()); + assertEquals(size1, metrics.getDbSnapshotsDirSize()); + assertEquals(2, metrics.getSstBackupSstFilesCount()); + assertEquals(backupSize1 + backupSize2, metrics.getSstBackupDirSize()); + + // Add another snapshot with a hardlink + Path snap2 = snapshotsDir.resolve(UUID.randomUUID().toString()); + Files.createDirectories(snap2); + Path file2 = snap2.resolve("file1" + ROCKSDB_SST_SUFFIX); + try { + Files.createLink(file2, file1); + } catch (UnsupportedOperationException e) { + // Fallback for systems that don't support hardlinks in temp dir + Files.write(file2, "data1".getBytes(StandardCharsets.UTF_8)); + } + + metrics.updateMetrics(); + + assertEquals(2, metrics.getNumSnapshots()); + // Total SST count should still be 1 if hardlink is counted once, + // but the implementation uses visitedInodes per snapshot directory? + // Wait, let's check the implementation again. + // calculateAndUpdateMetrics: visitedSnapshotsInodes is used for ALL snapshots. + assertEquals(1, metrics.getTotalSstFilesCount()); + assertEquals(size1, metrics.getDbSnapshotsDirSize()); + + // Test backup dir update + Path backupFile3 = backupDir.resolve("notSst.txt"); + Files.write(backupFile3, "notSstData".getBytes(StandardCharsets.UTF_8)); + + metrics.updateMetrics(); + assertEquals(2, metrics.getSstBackupSstFilesCount()); + assertEquals(backupSize1 + backupSize2 + Files.size(backupFile3), metrics.getSstBackupDirSize()); + } +} From e0a1c6da204ab202d9cad846cc24aedf2ef32485 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 9 Apr 2026 23:58:28 -0700 Subject: [PATCH 2/4] HDDS-14996. Fix checkstyle issues in TestOMSnapshotDirectoryMetrics. Change-Id: I0c50b044a6d44ea9d4eb0f96e4202d76998c9b41 --- .../TestOMSnapshotDirectoryMetrics.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java index 728909dd571f..5a86f01abfe0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java @@ -17,6 +17,16 @@ package org.apache.hadoop.ozone.om.snapshot; +import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -26,18 +36,6 @@ import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.UUID; - -import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; - /** * Tests for OMSnapshotDirectoryMetrics. */ From fc9354f7359917ee6ac53ffc677d370808446094 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 10 Apr 2026 00:18:19 -0700 Subject: [PATCH 3/4] HDDS-14996. Fix PMD SingularField warnings in TestOMSnapshotDirectoryMetrics. Change-Id: I162534c354312e0c33ee7fc4cabb646ac48780c8 --- .../ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java index 5a86f01abfe0..bb86ab895bbb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java @@ -45,15 +45,13 @@ public class TestOMSnapshotDirectoryMetrics { private Path tempDir; private OMMetadataManager metadataManager; - private RDBStore store; - private RocksDBCheckpointDiffer differ; private OMSnapshotDirectoryMetrics metrics; @BeforeEach public void setup() throws IOException { metadataManager = Mockito.mock(OMMetadataManager.class); - store = Mockito.mock(RDBStore.class); - differ = Mockito.mock(RocksDBCheckpointDiffer.class); + RDBStore store = Mockito.mock(RDBStore.class); + RocksDBCheckpointDiffer differ = Mockito.mock(RocksDBCheckpointDiffer.class); when(metadataManager.getStore()).thenReturn(store); when(store.getRocksDBCheckpointDiffer()).thenReturn(differ); From f3239c6aa7c6277e8ff1f84dad12cf20e0193946 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 10 Apr 2026 12:37:55 -0700 Subject: [PATCH 4/4] review commens Change-Id: I5b70076aa2549f9c2839acf873b8e316c1e85325 --- .../ozone/om/snapshot/OMSnapshotDirectoryMetrics.java | 7 ++++--- .../ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java | 4 ---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java index 5ab945c1ca2e..9afe5304284f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OMSnapshotDirectoryMetrics.java @@ -146,11 +146,12 @@ protected boolean updateMetrics() { } /** - * Calculates & updates directory size metrics accounting for hardlinks. - * (only counts each inode once). + * Calculates & updates directory size metrics accounting for snapshot and + * backup SST Directory. (only counts each inode once). * Uses Files.getAttribute to get the inode number and tracks visited inodes. * - * @param directory the directory containing all checkpointDirs. + * @param snapshotsDir the directory containing all checkpointDirs. + * @param sstBackupDir the backup SST directory */ private void calculateAndUpdateMetrics(File snapshotsDir, File sstBackupDir) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java index bb86ab895bbb..123a49ea20c3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOMSnapshotDirectoryMetrics.java @@ -111,10 +111,6 @@ public void testMetrics() throws IOException { metrics.updateMetrics(); assertEquals(2, metrics.getNumSnapshots()); - // Total SST count should still be 1 if hardlink is counted once, - // but the implementation uses visitedInodes per snapshot directory? - // Wait, let's check the implementation again. - // calculateAndUpdateMetrics: visitedSnapshotsInodes is used for ALL snapshots. assertEquals(1, metrics.getTotalSstFilesCount()); assertEquals(size1, metrics.getDbSnapshotsDirSize());