From 8c35336cae67c49c26a55e148ea5f741b592305b Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Tue, 2 Jun 2026 17:35:06 -0700 Subject: [PATCH 1/8] HDDS-15342. Make RocksDB BottommostLevelCompaction options configurable. --- .../common/src/main/resources/ozone-default.xml | 9 +++++++++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 9 +++++++++ .../apache/hadoop/ozone/om/OzoneManager.java | 13 ++++++++++++- .../hadoop/ozone/om/service/CompactDBUtil.java | 13 +++++++------ .../ozone/om/service/CompactionService.java | 17 +++++++++++++++-- .../ozone/om/service/TestCompactionService.java | 1 + 6 files changed, 53 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f9dce0b1e97b..a26f1383df4e 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1591,6 +1591,15 @@ If this is empty, no column families are compacted. + + ozone.om.compaction.service.bottommostlevelcompaction + 0 + OZONE, OM, PERFORMANCE + + Bottommost level compaction type for manual compaction. + Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce). + + ozone.om.snapshot.compact.non.snapshot.diff.tables diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index a43d3fb25982..1f5adbcf9635 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -681,6 +681,15 @@ public final class OMConfigKeys { public static final String OZONE_OM_COMPACTION_SERVICE_COLUMNFAMILIES_DEFAULT = "keyTable,fileTable,directoryTable,deletedTable,deletedDirectoryTable,multipartInfoTable,multipartPartsTable"; + /** + * Bottommost level compaction type for manual compaction. + * Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce). + * Refer to {@code org.rocksdb.CompactRangeOptions.BottommostLevelCompaction}. + */ + public static final String OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION = + "ozone.om.compaction.service.bottommostlevelcompaction"; + public static final String OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT = "0"; + /** * Configuration to enable/disable non-snapshot diff table compaction when snapshots are evicted from cache. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index d258324b3bb0..dce70ef375f1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -60,6 +60,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INTERVAL_MS_DEFAULT; @@ -221,6 +222,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc_.ProtobufRpcEngine; import org.apache.hadoop.ipc_.RPC; @@ -5644,8 +5646,17 @@ public void checkFeatureEnabled(OzoneManagerVersion feature) throws OMException public void compactOMDB(String columnFamily) throws IOException { checkAdminUserPrivilege("compact column family " + columnFamily); + int compactionType = Integer.parseInt(configuration.get( + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, + OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction = + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); + if (bottommostLevelCompaction == null) { + compactionType = Integer.parseInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); + bottommostLevelCompaction = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); + } CompletableFuture compactFuture = - CompactDBUtil.compactTableAsync(metadataManager, columnFamily); + CompactDBUtil.compactTableAsync(metadataManager, columnFamily, bottommostLevelCompaction); compactFuture.whenComplete((result, throwable) -> { if (throwable == null) { LOG.info("Compaction request for column family \"{}\" completed successfully.", diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java index 6eb52abbcfa0..5f488374fc61 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java @@ -38,13 +38,13 @@ public final class CompactDBUtil { private CompactDBUtil() { } - public static void compactTable(OMMetadataManager omMetadataManager, - String tableName) throws IOException { + public static void compactTable(OMMetadataManager omMetadataManager, String tableName, + ManagedCompactRangeOptions.BottommostLevelCompaction compactionType) throws IOException { long startTime = Time.monotonicNow(); LOG.info("Compacting column family: {}", tableName); try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { - options.setBottommostLevelCompaction( - ManagedCompactRangeOptions.BottommostLevelCompaction.kForce); + options.setBottommostLevelCompaction(compactionType); + LOG.debug("Setting bottommost level compaction type to {}", options.bottommostLevelCompaction()); options.setExclusiveManualCompaction(true); RocksDatabase rocksDatabase = ((RDBStore) omMetadataManager.getStore()).getDb(); @@ -62,10 +62,11 @@ public static void compactTable(OMMetadataManager omMetadataManager, } } - public static CompletableFuture compactTableAsync(OMMetadataManager metadataManager, String tableName) { + public static CompletableFuture compactTableAsync(OMMetadataManager metadataManager, String tableName, + ManagedCompactRangeOptions.BottommostLevelCompaction compactionType) { return CompletableFuture.runAsync(() -> { try { - compactTable(metadataManager, tableName); + compactTable(metadataManager, tableName, compactionType); } catch (Exception e) { LOG.warn("Failed to compact column family: {}", tableName, e); throw new CompletionException("Compaction failed for column family: " + tableName, e); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java index ec24fe117bbe..aeb91655aa8e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -32,6 +32,8 @@ import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.slf4j.Logger; @@ -53,6 +55,7 @@ public class CompactionService extends BackgroundService { private final AtomicBoolean suspended; // list of tables that can be compacted private final List compactableTables; + private final ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction; public CompactionService(OzoneManager ozoneManager, TimeUnit unit, long interval, long timeout, List tables) { @@ -65,6 +68,16 @@ public CompactionService(OzoneManager ozoneManager, TimeUnit unit, long interval this.numCompactions = new AtomicLong(0); this.suspended = new AtomicBoolean(false); this.compactableTables = validateTables(tables); + int compactionType = Integer.parseInt(ozoneManager.getConfiguration().get( + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + ManagedCompactRangeOptions.BottommostLevelCompaction level = + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); + if (level == null) { + compactionType = Integer.parseInt(OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); + level = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); + } + this.bottommostLevelCompaction = level; } private List validateTables(List tables) { @@ -141,11 +154,11 @@ private boolean shouldRun() { * @return CompletableFuture that completes when compaction finishes */ public CompletableFuture compactTableAsync(String tableName) { - return CompactDBUtil.compactTableAsync(omMetadataManager, tableName); + return CompactDBUtil.compactTableAsync(omMetadataManager, tableName, bottommostLevelCompaction); } protected void compactFully(String tableName) throws IOException { - CompactDBUtil.compactTable(omMetadataManager, tableName); + CompactDBUtil.compactTable(omMetadataManager, tableName, bottommostLevelCompaction); } private class CompactTask implements BackgroundTask { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java index d25423156701..62ebdb55e733 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java @@ -74,6 +74,7 @@ void setup(@TempDir Path tempDir) { ozoneManager = mock(OzoneManager.class); OMMetadataManager metadataManager = mock(OMMetadataManager.class); when(ozoneManager.getMetadataManager()).thenReturn(metadataManager); + when(ozoneManager.getConfiguration()).thenReturn(conf); TypedTable table = mock(TypedTable.class); Set tables = new HashSet<>(); From 728195326332a3787c465e00130ed118ba7a76d6 Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Tue, 2 Jun 2026 18:00:52 -0700 Subject: [PATCH 2/8] HDDS-15342. Add unit tests for RocksDB BottommostLevelCompaction configurability. --- .../ozone/om/service/CompactionService.java | 5 + .../ozone/om/service/TestCompactDBUtil.java | 118 ++++++++++++++++++ .../om/service/TestCompactionService.java | 32 +++++ 3 files changed, 155 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java index aeb91655aa8e..95fc5d2983fc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -121,6 +121,11 @@ public List getCompactableTables() { return compactableTables; } + @VisibleForTesting + public ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction() { + return bottommostLevelCompaction; + } + /** * Returns the number of manual compactions performed. * diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java new file mode 100644 index 000000000000..4cca7bd1467d --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.service; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.File; +import java.io.IOException; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link CompactDBUtil}. + */ +class TestCompactDBUtil { + + private OMMetadataManager omMetadataManager; + + @BeforeEach + void setup(@TempDir File tempDir) throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OZONE_OM_DB_DIRS, tempDir.getAbsolutePath()); + omMetadataManager = new OmMetadataManagerImpl(conf, null); + } + + @Test + void testCompactWithKSkip() { + assertDoesNotThrow(() -> + CompactDBUtil.compactTable(omMetadataManager, "keyTable", + ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip)); + } + + @Test + void testCompactWithKForce() { + assertDoesNotThrow(() -> + CompactDBUtil.compactTable(omMetadataManager, "keyTable", + ManagedCompactRangeOptions.BottommostLevelCompaction.kForce)); + } + + @Test + void testCompactWithKIfHaveCompactionFilter() { + assertDoesNotThrow(() -> + CompactDBUtil.compactTable(omMetadataManager, "keyTable", + ManagedCompactRangeOptions.BottommostLevelCompaction.kIfHaveCompactionFilter)); + } + + @Test + void testCompactInvalidColumnFamily() { + assertThrows(IOException.class, () -> + CompactDBUtil.compactTable(omMetadataManager, "nonExistentTable", + ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip)); + } + + @Test + void testDefaultConfigValueMapsToKSkip() { + int defaultValue = Integer.parseInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); + assertEquals(0, defaultValue); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(defaultValue)); + } + + @Test + void testConfigValueMapsToCorrectEnum() { + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(0)); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kIfHaveCompactionFilter, + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(1)); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce, + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(2)); + } + + @Test + void testInvalidConfigValueReturnsNull() { + assertNull(ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(99)); + } + + @Test + void testConfigKeyIsReadFromOzoneConfiguration() { + OzoneConfiguration conf = new OzoneConfiguration(); + // Default - not set, should use the default value + assertEquals("0", conf.get(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, + OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + + // Override to kForce + conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "2"); + int compactionType = Integer.parseInt(conf.get( + OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, + OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce, + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType)); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java index 62ebdb55e733..d6fe199b16ef 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java @@ -17,8 +17,10 @@ package org.apache.hadoop.ozone.om.service; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_RUN_INTERVAL; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -39,6 +41,7 @@ import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; import org.apache.hadoop.hdds.utils.db.TypedTable; +import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -160,6 +163,35 @@ public void testCompactFailure() { () -> getCompactionService(compactTables)); } + @Test + public void testDefaultCompactionLevelIsKSkip() { + CompactionService compactionService = getCompactionService(Arrays.asList("keyTable", "fileTable")); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, + compactionService.getBottommostLevelCompaction()); + } + + @Test + public void testConfiguredCompactionLevelKForce() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "2"); + when(ozoneManager.getConfiguration()).thenReturn(conf); + + CompactionService compactionService = getCompactionService(Arrays.asList("keyTable", "fileTable")); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce, + compactionService.getBottommostLevelCompaction()); + } + + @Test + public void testInvalidCompactionLevelFallsBackToDefault() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "99"); + when(ozoneManager.getConfiguration()).thenReturn(conf); + + CompactionService compactionService = getCompactionService(Arrays.asList("keyTable", "fileTable")); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, + compactionService.getBottommostLevelCompaction()); + } + private CompactionService getCompactionService(List compactTables) { CompactionService compactionService = new CompactionService(ozoneManager, TimeUnit.MILLISECONDS, TimeUnit.SECONDS.toMillis(SERVICE_INTERVAL), TimeUnit.SECONDS.toMillis(60), compactTables) { From 059ac20dd8612740100b0838b273ffe6df6c8245 Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Mon, 8 Jun 2026 20:10:01 -0700 Subject: [PATCH 3/8] HDDS-15342. Refactor to centralize RocksDB BottommostLevelCompaction configuration logic. --- .../apache/hadoop/ozone/om/OzoneManager.java | 10 +-------- .../ozone/om/service/CompactDBUtil.java | 21 +++++++++++++++++-- .../ozone/om/service/CompactionService.java | 12 +---------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index dce70ef375f1..9c1f32e28cdf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -60,7 +60,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_EDEKCACHELOADER_INTERVAL_MS_DEFAULT; @@ -5646,15 +5645,8 @@ public void checkFeatureEnabled(OzoneManagerVersion feature) throws OMException public void compactOMDB(String columnFamily) throws IOException { checkAdminUserPrivilege("compact column family " + columnFamily); - int compactionType = Integer.parseInt(configuration.get( - OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, - OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction = - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); - if (bottommostLevelCompaction == null) { - compactionType = Integer.parseInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); - bottommostLevelCompaction = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); - } + CompactDBUtil.getBottommostLevelCompaction(configuration); CompletableFuture compactFuture = CompactDBUtil.compactTableAsync(metadataManager, columnFamily, bottommostLevelCompaction); compactFuture.whenComplete((result, throwable) -> { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java index 5f488374fc61..7fcb5a08638c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java @@ -20,9 +20,11 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.RocksDatabase; import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.util.Time; import org.slf4j.Logger; @@ -41,10 +43,10 @@ private CompactDBUtil() { public static void compactTable(OMMetadataManager omMetadataManager, String tableName, ManagedCompactRangeOptions.BottommostLevelCompaction compactionType) throws IOException { long startTime = Time.monotonicNow(); - LOG.info("Compacting column family: {}", tableName); try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { options.setBottommostLevelCompaction(compactionType); - LOG.debug("Setting bottommost level compaction type to {}", options.bottommostLevelCompaction()); + LOG.info("Compacting column family: {} with {} bottommostlevel compaction", + tableName, options.bottommostLevelCompaction()); options.setExclusiveManualCompaction(true); RocksDatabase rocksDatabase = ((RDBStore) omMetadataManager.getStore()).getDb(); @@ -73,4 +75,19 @@ public static CompletableFuture compactTableAsync(OMMetadataManager metada } }); } + + public static ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction( + OzoneConfiguration configuration) { + int compactionType = Integer.parseInt(configuration.get( + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + ManagedCompactRangeOptions.BottommostLevelCompaction level = + ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); + if (level == null) { + compactionType = Integer.parseInt(OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); + level = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); + LOG.warn("Invalid bottommost level compaction type. Using default value: {}", level); + } + return level; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java index 95fc5d2983fc..9c4653110c78 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; -import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.slf4j.Logger; @@ -68,16 +67,7 @@ public CompactionService(OzoneManager ozoneManager, TimeUnit unit, long interval this.numCompactions = new AtomicLong(0); this.suspended = new AtomicBoolean(false); this.compactableTables = validateTables(tables); - int compactionType = Integer.parseInt(ozoneManager.getConfiguration().get( - OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, - OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); - ManagedCompactRangeOptions.BottommostLevelCompaction level = - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); - if (level == null) { - compactionType = Integer.parseInt(OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); - level = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); - } - this.bottommostLevelCompaction = level; + this.bottommostLevelCompaction = CompactDBUtil.getBottommostLevelCompaction(ozoneManager.getConfiguration()); } private List validateTables(List tables) { From 422c85c850a265025ace2720939b6f81a46d331d Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Tue, 9 Jun 2026 11:52:13 -0700 Subject: [PATCH 4/8] HDDS-15342. Refactor to use `getInt` for BottommostLevelCompaction configuration and update its default type. --- .../java/org/apache/hadoop/ozone/om/OMConfigKeys.java | 2 +- .../apache/hadoop/ozone/om/service/CompactDBUtil.java | 8 ++++---- .../hadoop/ozone/om/service/TestCompactDBUtil.java | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 1f5adbcf9635..dc4c8542278c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -688,7 +688,7 @@ public final class OMConfigKeys { */ public static final String OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION = "ozone.om.compaction.service.bottommostlevelcompaction"; - public static final String OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT = "0"; + public static final int OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT = 0; /** * Configuration to enable/disable non-snapshot diff table compaction when snapshots are evicted from cache. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java index 7fcb5a08638c..88d60c1f70f0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java @@ -45,7 +45,7 @@ public static void compactTable(OMMetadataManager omMetadataManager, String tabl long startTime = Time.monotonicNow(); try (ManagedCompactRangeOptions options = new ManagedCompactRangeOptions()) { options.setBottommostLevelCompaction(compactionType); - LOG.info("Compacting column family: {} with {} bottommostlevel compaction", + LOG.info("Compacting column family: {} with {} bottommost level compaction", tableName, options.bottommostLevelCompaction()); options.setExclusiveManualCompaction(true); RocksDatabase rocksDatabase = @@ -78,13 +78,13 @@ public static CompletableFuture compactTableAsync(OMMetadataManager metada public static ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction( OzoneConfiguration configuration) { - int compactionType = Integer.parseInt(configuration.get( + int compactionType = configuration.getInt( OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, - OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); ManagedCompactRangeOptions.BottommostLevelCompaction level = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); if (level == null) { - compactionType = Integer.parseInt(OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); + compactionType = OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; level = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); LOG.warn("Invalid bottommost level compaction type. Using default value: {}", level); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java index 4cca7bd1467d..9819169bfa35 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java @@ -79,7 +79,7 @@ void testCompactInvalidColumnFamily() { @Test void testDefaultConfigValueMapsToKSkip() { - int defaultValue = Integer.parseInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); + int defaultValue = OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; assertEquals(0, defaultValue); assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(defaultValue)); @@ -104,14 +104,14 @@ void testInvalidConfigValueReturnsNull() { void testConfigKeyIsReadFromOzoneConfiguration() { OzoneConfiguration conf = new OzoneConfiguration(); // Default - not set, should use the default value - assertEquals("0", conf.get(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, + assertEquals(0, conf.getInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); // Override to kForce - conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "2"); - int compactionType = Integer.parseInt(conf.get( + conf.setInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, 2); + int compactionType = conf.getInt( OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, - OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce, ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType)); } From d216a87a4b1f4c8466734830a07429d7539582c3 Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Thu, 11 Jun 2026 10:52:07 -0700 Subject: [PATCH 5/8] HDDS-15342. Update compaction configuration documentation --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 5 +++-- .../main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java | 3 ++- .../hadoop/ozone/om/service/TestCompactionService.java | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index a26f1383df4e..13866f96c7e3 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1596,8 +1596,9 @@ 0 OZONE, OM, PERFORMANCE - Bottommost level compaction type for manual compaction. - Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce). + Bottommost level compaction type for compaction. + Invalid values will default to kSkip. + Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce), 3 (kForceOptimized). diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index dc4c8542278c..9199bd16cd7d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -683,7 +683,8 @@ public final class OMConfigKeys { /** * Bottommost level compaction type for manual compaction. - * Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce). + * Invalid values will default to kSkip. + * Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce), 3 (kForceOptimized). * Refer to {@code org.rocksdb.CompactRangeOptions.BottommostLevelCompaction}. */ public static final String OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java index d6fe199b16ef..5fda4b4a0e35 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java @@ -173,7 +173,7 @@ public void testDefaultCompactionLevelIsKSkip() { @Test public void testConfiguredCompactionLevelKForce() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "2"); + conf.setInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, 2); when(ozoneManager.getConfiguration()).thenReturn(conf); CompactionService compactionService = getCompactionService(Arrays.asList("keyTable", "fileTable")); @@ -184,7 +184,7 @@ public void testConfiguredCompactionLevelKForce() { @Test public void testInvalidCompactionLevelFallsBackToDefault() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "99"); + conf.setInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, 99); when(ozoneManager.getConfiguration()).thenReturn(conf); CompactionService compactionService = getCompactionService(Arrays.asList("keyTable", "fileTable")); From d74e0747eb4e10be8fdfd254b1c644f6b5a9fa98 Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Wed, 17 Jun 2026 10:43:15 -0700 Subject: [PATCH 6/8] HDDS-15342. Remove redundant BottommostLevelCompaction initialization in OzoneManager. --- .../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 344e3c74ed43..db7717c81ea2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -5667,8 +5667,6 @@ public void checkFeatureEnabled(OzoneManagerVersion feature) throws OMException public void compactOMDB(String columnFamily, ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction) throws IOException { checkAdminUserPrivilege("compact column family " + columnFamily); - ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction = - CompactDBUtil.getBottommostLevelCompaction(configuration); CompletableFuture compactFuture = CompactDBUtil.compactTableAsync(metadataManager, columnFamily, bottommostLevelCompaction); compactFuture.whenComplete((result, throwable) -> { From 02e431ee5a554e002a3318a16c7c50b593d3600f Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Mon, 13 Jul 2026 17:47:26 -0700 Subject: [PATCH 7/8] HDDS-15342. Refactor tests and improve handling of invalid or custom BottommostLevelCompaction configuration values. --- hadoop-ozone/common/pom.xml | 4 ++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 5 +- .../ozone/om/service/CompactDBUtil.java | 22 +++--- .../ozone/om/service/TestCompactDBUtil.java | 72 ++++++------------- 4 files changed, 43 insertions(+), 60 deletions(-) diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index a5318a66f838..d3fe5875630d 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -105,6 +105,10 @@ org.apache.ozone hdds-interface-client + + org.apache.ozone + hdds-managed-rocksdb + org.apache.ozone ozone-interface-client diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 6e404e5fdafb..168b411ce398 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -20,6 +20,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.ratis.util.TimeDuration; @@ -689,7 +690,9 @@ public final class OMConfigKeys { */ public static final String OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION = "ozone.om.compaction.service.bottommostlevelcompaction"; - public static final int OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT = 0; + public static final ManagedCompactRangeOptions.BottommostLevelCompaction + OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT = + ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip; /** * Configuration to enable/disable non-snapshot diff table compaction when snapshots are evicted from cache. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java index 838399ecf90c..d21a655cf9b9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactDBUtil.java @@ -78,17 +78,19 @@ public static CompletableFuture compactTableAsync(OMMetadataManager metada public static ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction( OzoneConfiguration configuration) { - int compactionType = configuration.getInt( - OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, - OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); - ManagedCompactRangeOptions.BottommostLevelCompaction level = - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); - if (level == null) { - compactionType = OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; - level = ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType); - LOG.warn("Invalid bottommost level compaction type. Using default value: {}", level); + ManagedCompactRangeOptions.BottommostLevelCompaction blc = + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; + + try { + blc = configuration.getEnum( + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, + OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); + } catch (IllegalArgumentException e) { + LOG.warn("Invalid value for bottommost level compaction configuration '{}'", + configuration.get(OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION), e); } - return level; + + return blc; } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java index 9819169bfa35..f757549c8ebe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactDBUtil.java @@ -20,9 +20,9 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.om.service.CompactDBUtil.getBottommostLevelCompaction; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.File; @@ -34,6 +34,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; /** * Tests for {@link CompactDBUtil}. @@ -49,25 +52,11 @@ void setup(@TempDir File tempDir) throws Exception { omMetadataManager = new OmMetadataManagerImpl(conf, null); } - @Test - void testCompactWithKSkip() { - assertDoesNotThrow(() -> - CompactDBUtil.compactTable(omMetadataManager, "keyTable", - ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip)); - } - - @Test - void testCompactWithKForce() { + @ParameterizedTest + @EnumSource(ManagedCompactRangeOptions.BottommostLevelCompaction.class) + void testCompactionAlgorithms(ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction) { assertDoesNotThrow(() -> - CompactDBUtil.compactTable(omMetadataManager, "keyTable", - ManagedCompactRangeOptions.BottommostLevelCompaction.kForce)); - } - - @Test - void testCompactWithKIfHaveCompactionFilter() { - assertDoesNotThrow(() -> - CompactDBUtil.compactTable(omMetadataManager, "keyTable", - ManagedCompactRangeOptions.BottommostLevelCompaction.kIfHaveCompactionFilter)); + CompactDBUtil.compactTable(omMetadataManager, "keyTable", bottommostLevelCompaction)); } @Test @@ -79,40 +68,25 @@ void testCompactInvalidColumnFamily() { @Test void testDefaultConfigValueMapsToKSkip() { - int defaultValue = OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT; - assertEquals(0, defaultValue); assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(defaultValue)); - } - - @Test - void testConfigValueMapsToCorrectEnum() { - assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(0)); - assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kIfHaveCompactionFilter, - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(1)); - assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce, - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(2)); - } - - @Test - void testInvalidConfigValueReturnsNull() { - assertNull(ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(99)); + OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); } - @Test - void testConfigKeyIsReadFromOzoneConfiguration() { + @ParameterizedTest + @ValueSource(strings = {"", "kForceeee"}) + void testDefaultConfigKeyIsReadFromOzoneConfiguration(String compactionType) { + // unset or invalid values should use the default value OzoneConfiguration conf = new OzoneConfiguration(); - // Default - not set, should use the default value - assertEquals(0, conf.getInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, - OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT)); + conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, compactionType); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kSkip, getBottommostLevelCompaction(conf)); + } - // Override to kForce - conf.setInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, 2); - int compactionType = conf.getInt( - OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, - OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION_DEFAULT); - assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce, - ManagedCompactRangeOptions.BottommostLevelCompaction.fromRocksId(compactionType)); + @ParameterizedTest + @ValueSource(strings = {"kForce", " kForce", "kForce ", " kForce "}) + void testConfigKeyIsReadFromOzoneConfiguration(String compactionType) { + // have trailing spaces in the config values to ensure they are trimmed and handled correctly + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, compactionType); + assertEquals(ManagedCompactRangeOptions.BottommostLevelCompaction.kForce, getBottommostLevelCompaction(conf)); } } From ec91d50522333673cfa95284de03753c0f4d9a73 Mon Sep 17 00:00:00 2001 From: Rishabh Patel Date: Wed, 15 Jul 2026 20:36:10 -0700 Subject: [PATCH 8/8] HDDS-15342. Refactor to replace integer-based BottommostLevelCompaction with enum and update configuration handling. --- .../src/main/resources/ozone-default.xml | 4 ++-- .../repair/ldb/RocksDBManualCompaction.java | 15 ++++++--------- .../hadoop/ozone/repair/om/CompactOMDB.java | 14 ++++++-------- hadoop-ozone/common/pom.xml | 18 ++++++++++++++++++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 2 +- .../ozone/om/service/CompactionService.java | 3 +-- .../om/service/TestCompactionService.java | 4 ++-- 7 files changed, 36 insertions(+), 24 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 068520e736ee..6394f101768f 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1593,12 +1593,12 @@ ozone.om.compaction.service.bottommostlevelcompaction - 0 + kSkip OZONE, OM, PERFORMANCE Bottommost level compaction type for compaction. Invalid values will default to kSkip. - Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce), 3 (kForceOptimized). + Valid values: kSkip, kIfHaveCompactionFilter, kForce, kForceOptimized. diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java index 0b89bb42418f..1a2373b08fa9 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/ldb/RocksDBManualCompaction.java @@ -27,7 +27,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.debug.RocksDBUtils; -import org.apache.hadoop.ozone.om.service.CompactDBUtil; import org.apache.hadoop.ozone.repair.RepairTool; import org.apache.hadoop.util.Time; import org.rocksdb.ColumnFamilyDescriptor; @@ -64,11 +63,11 @@ public class RocksDBManualCompaction extends RepairTool { private String columnFamilyName; @CommandLine.Option(names = {"--bottommost-level-compaction", "--blc"}, - description = "BottommostLevelCompaction option for RocksDB compaction." + - " Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce), 3 (kForceOptimized).", - defaultValue = "0", + description = "BottommostLevelCompaction algorithm for RocksDB compaction." + + " Valid values: ${COMPLETION-CANDIDATES}", + defaultValue = "kSkip", showDefaultValue = CommandLine.Help.Visibility.ALWAYS) - private int bottommostLevelCompaction; + private ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction; private String getConsoleReadLineWithFormat() { err().printf(WARNING_TO_STOP_SERVICE); @@ -104,14 +103,12 @@ public void execute() throws Exception { " is not in a column family in DB for the given path."); } - ManagedCompactRangeOptions.BottommostLevelCompaction blcOption = - CompactDBUtil.getBottommostLevelCompaction(bottommostLevelCompaction); info("Running compaction on " + columnFamilyName + - " with bottommost level compaction: " + blcOption.name()); + " with bottommost level compaction: " + bottommostLevelCompaction.name()); long startTime = Time.monotonicNow(); if (!isDryRun()) { ManagedCompactRangeOptions compactOptions = new ManagedCompactRangeOptions(); - compactOptions.setBottommostLevelCompaction(blcOption); + compactOptions.setBottommostLevelCompaction(bottommostLevelCompaction); db.get().compactRange(cfh, null, null, compactOptions); } long duration = Time.monotonicNow() - startTime; diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java index 8e6c04ddb08c..4976b4cf8fd0 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/CompactOMDB.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedCompactRangeOptions; import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; import org.apache.hadoop.ozone.om.protocolPB.OMAdminProtocolClientSideImpl; -import org.apache.hadoop.ozone.om.service.CompactDBUtil; import org.apache.hadoop.ozone.repair.RepairTool; import org.apache.hadoop.security.UserGroupInformation; import picocli.CommandLine; @@ -62,10 +61,10 @@ public class CompactOMDB extends RepairTool { @CommandLine.Option(names = {"--bottommost-level-compaction", "--blc"}, description = "BottommostLevelCompaction option for RocksDB compaction." + - " Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce), 3 (kForceOptimized).", - defaultValue = "0", + " Valid values: ${COMPLETION-CANDIDATES}", + defaultValue = "kSkip", showDefaultValue = CommandLine.Help.Visibility.ALWAYS) - private int bottommostLevelCompaction; + private ManagedCompactRangeOptions.BottommostLevelCompaction bottommostLevelCompaction; @Override public void execute() throws Exception { @@ -81,15 +80,14 @@ public void execute() throws Exception { } String omDisplay = nodeId != null ? nodeId : omNodeDetails.getRpcAddressString(); - ManagedCompactRangeOptions.BottommostLevelCompaction blcOption = - CompactDBUtil.getBottommostLevelCompaction(bottommostLevelCompaction); if (!isDryRun()) { try (OMAdminProtocolClientSideImpl omAdminProtocolClient = OMAdminProtocolClientSideImpl.createProxyForSingleOM(conf, UserGroupInformation.getCurrentUser(), omNodeDetails)) { - omAdminProtocolClient.compactOMDB(columnFamilyName, blcOption.getValue()); + omAdminProtocolClient.compactOMDB(columnFamilyName, bottommostLevelCompaction.getValue()); info("Compaction request issued for om.db of om node: %s, column-family: %s" + - " with bottommost level compaction: %s.", omDisplay, columnFamilyName, blcOption.name()); + " with bottommost level compaction: %s.", + omDisplay, columnFamilyName, bottommostLevelCompaction.name()); info("Please check role logs of %s for completion status.", omDisplay); } catch (IOException ex) { error("Couldn't compact column %s. \nException: %s", columnFamilyName, ex); diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index d3fe5875630d..9e6b5fd37b78 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -125,6 +125,10 @@ org.apache.ratis ratis-thirdparty-misc + + org.rocksdb + rocksdbjni + org.slf4j slf4j-api @@ -231,6 +235,20 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.apache.ozone:hdds-managed-rocksdb:jar + + + org.apache.maven.plugins maven-enforcer-plugin diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 168b411ce398..379991f65045 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -685,7 +685,7 @@ public final class OMConfigKeys { /** * Bottommost level compaction type for manual compaction. * Invalid values will default to kSkip. - * Valid values: 0 (kSkip), 1 (kIfHaveCompactionFilter), 2 (kForce), 3 (kForceOptimized). + * Valid values: kSkip, kIfHaveCompactionFilter, kForce, kForceOptimized. * Refer to {@code org.rocksdb.CompactRangeOptions.BottommostLevelCompaction}. */ public static final String OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java index 9c4653110c78..053efcbace63 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/CompactionService.java @@ -111,8 +111,7 @@ public List getCompactableTables() { return compactableTables; } - @VisibleForTesting - public ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction() { + ManagedCompactRangeOptions.BottommostLevelCompaction getBottommostLevelCompaction() { return bottommostLevelCompaction; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java index 5fda4b4a0e35..4299abd6253e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestCompactionService.java @@ -173,7 +173,7 @@ public void testDefaultCompactionLevelIsKSkip() { @Test public void testConfiguredCompactionLevelKForce() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, 2); + conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "kForce"); when(ozoneManager.getConfiguration()).thenReturn(conf); CompactionService compactionService = getCompactionService(Arrays.asList("keyTable", "fileTable")); @@ -184,7 +184,7 @@ public void testConfiguredCompactionLevelKForce() { @Test public void testInvalidCompactionLevelFallsBackToDefault() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, 99); + conf.set(OZONE_OM_COMPACTION_SERVICE_BOTTOMMOSTLEVELCOMPACTION, "kForceeeee"); when(ozoneManager.getConfiguration()).thenReturn(conf); CompactionService compactionService = getCompactionService(Arrays.asList("keyTable", "fileTable"));