From 06d15e33431281c232d9db0a9bd33201e055d24e Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 May 2026 11:47:09 +0100 Subject: [PATCH 01/37] HDDS-15208. OM should learn to finalize from SCM --- .../apache/hadoop/ozone/om/OMConfigKeys.java | 4 + .../hadoop/fs/ozone/TestHSyncUpgrade.java | 36 ++-- .../ozone/om/TestOMBucketLayoutUpgrade.java | 10 +- .../ozone/om/TestOMUpgradeFinalization.java | 14 +- .../om/multitenant/TestMultiTenantVolume.java | 35 +--- .../ozone/om/snapshot/TestOmSnapshot.java | 26 +-- .../apache/hadoop/ozone/om/OzoneManager.java | 16 ++ .../om/upgrade/OMUpgradeFinalizeService.java | 120 +++++++++++ .../upgrade/TestOMUpgradeFinalizeService.java | 186 ++++++++++++++++++ 9 files changed, 367 insertions(+), 80 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index b9ccfedb40d0..a9ec011ac07a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -698,6 +698,10 @@ public final class OMConfigKeys { "ozone.om.ratis.events.max.limit"; public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100; + public static final String OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL = + "ozone.om.upgrade.finalization.check.interval"; + public static final String OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL_DEFAULT = "1m"; + /** * Never constructed. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java index 22d8f60920fa..e563cba60528 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java @@ -29,16 +29,11 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; -import static org.apache.ozone.test.LambdaTestUtils.await; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; -import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataOutputStream; @@ -46,26 +41,29 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.storage.BlockInputStream; import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.scm.storage.BufferPool; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl; import org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OMUpgradeTestUtils; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -93,9 +91,6 @@ public class TestHSyncUpgrade { private static final int SERVICE_INTERVAL = 100; private static final int EXPIRE_THRESHOLD_MS = 140; - private static final int POLL_INTERVAL_MILLIS = 500; - private static final int POLL_MAX_WAIT_MILLIS = 120_000; - @BeforeEach public void init() throws Exception { final BucketLayout layout = BUCKET_LAYOUT; @@ -117,7 +112,10 @@ public void init() throws Exception { conf.setTimeDuration(OZONE_OM_LEASE_HARD_LIMIT, EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s"); - conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion()); + conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OzoneManagerVersion.ATOMIC_REWRITE_KEY.serialize()); + conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, + HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) @@ -217,20 +215,8 @@ private void finalizeOMUpgrade() throws Exception { // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); - UpgradeFinalization.StatusAndMessages finalizationResponse = - omClient.finalizeUpgrade(upgradeClientID); - - // The status should transition as soon as the client call above returns - assertTrue(isStarting(finalizationResponse.status())); - // Wait for the finalization to be marked as done. - // 10s timeout should be plenty. - await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> { - final UpgradeFinalization.StatusAndMessages progress = - omClient.queryUpgradeFinalizationProgress( - upgradeClientID, false, false); - return isDone(progress.status()); - }); + cluster.getStorageContainerLocationClient().finalizeUpgrade(); + OMUpgradeTestUtils.waitForFinalization(omClient); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index 3f4120a48420..fe5a7629a01e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -32,6 +32,8 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; @@ -42,7 +44,6 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.ozone.test.LambdaTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -89,6 +90,8 @@ class TestOMBucketLayoutUpgrade { void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, fromVersion.serialize()); + conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); String omServiceId = UUID.randomUUID().toString(); MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf); builder.setOMServiceId(omServiceId) @@ -152,10 +155,7 @@ void allowsLegacyBucketBeforeUpgrade() throws Exception { @Test @Order(DURING_UPGRADE) void finalizeUpgrade() throws Exception { - UpgradeFinalization.StatusAndMessages response = - omClient.finalizeUpgrade("finalize-test"); - System.out.println("Finalization Messages : " + response.msgs()); - + cluster.getStorageContainerLocationClient().finalizeUpgrade(); waitForFinalization(omClient); final String expectedVersion = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index 476ae16af055..3c04493d213b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -31,6 +31,8 @@ import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneManagerVersion; @@ -40,7 +42,6 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages; import org.apache.ratis.util.LifeCycle; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -69,6 +70,11 @@ public static void shutdown() { @Test void testOMUpgradeFinalizationWithOneOMDown() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, INITIAL_VERSION.layoutVersion()); + // Finalization is triggered when SCM is finalized, so we first start it unfinalized and then finalize it to trigger + // the OM finalization. + conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); try (MiniOzoneHAClusterImpl cluster = newCluster(conf)) { cluster.waitForClusterToBeReady(); @@ -93,12 +99,10 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { long prepareIndex = omClient.prepareOzoneManager(120L, 5L); assertClusterPrepared(prepareIndex, runningOms); AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_PREPARE, AuditEventStatus.SUCCESS); - omClient.cancelOzoneManagerPrepare(); AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_CANCEL, AuditEventStatus.SUCCESS); - StatusAndMessages response = - omClient.finalizeUpgrade("finalize-test"); - System.out.println("Finalization Messages : " + response.msgs()); + // Send the finalize command to SCM which triggers the OM finalize when SCM reports it is complete. + cluster.getStorageContainerLocationClient().finalizeUpgrade(); AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); waitForFinalization(omClient); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index f1102777327c..89d3823b93e9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@ -18,13 +18,9 @@ package org.apache.hadoop.ozone.om.multitenant; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MULTITENANCY_ENABLED; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import com.google.protobuf.ServiceException; import java.io.IOException; @@ -33,6 +29,8 @@ import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -41,15 +39,15 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.rpc.RpcClient; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl; import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OMUpgradeTestUtils; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.protocol.S3Auth; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; -import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.LambdaTestUtils.VoidCallable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -77,6 +75,8 @@ public static void initClusterProvider() throws Exception { OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true); conf.setBoolean(OZONE_OM_MULTITENANCY_ENABLED, true); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.INITIAL_VERSION.layoutVersion()); + conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf) .withoutDatanodes(); cluster = builder.build(); @@ -146,27 +146,8 @@ private static void finalizeOMUpgrade() // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); - UpgradeFinalization.StatusAndMessages finalizationResponse = - omClient.finalizeUpgrade(upgradeClientID); - - // The status should transition as soon as the client call above returns - assertTrue(isStarting(finalizationResponse.status())); - - // Wait for the finalization to be marked as done. - // 10s timeout should be plenty. - GenericTestUtils.waitFor(() -> { - try { - final UpgradeFinalization.StatusAndMessages progress = - omClient.queryUpgradeFinalizationProgress( - upgradeClientID, false, false); - return isDone(progress.status()); - } catch (IOException e) { - fail("Unexpected exception while waiting for " - + "the OM upgrade to finalize: " + e.getMessage()); - } - return false; - }, 500, 10000); + cluster.getStorageContainerLocationClient().finalizeUpgrade(); + OMUpgradeTestUtils.waitForFinalization(omClient); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index becf7fb9e5ac..36547377714c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -50,8 +50,6 @@ import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG; import static org.apache.ozone.test.LambdaTestUtils.await; import static org.assertj.core.api.Assertions.assertThat; @@ -101,6 +99,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; +import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.db.CodecBuffer; import org.apache.hadoop.hdds.utils.db.DBProfile; import org.apache.hadoop.hdds.utils.db.DBStore; @@ -134,6 +134,7 @@ import org.apache.hadoop.ozone.om.KeyManagerImpl; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OMUpgradeTestUtils; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -153,7 +154,6 @@ import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.ozone.compaction.log.CompactionLogEntry; @@ -234,6 +234,9 @@ private void init() throws Exception { // Enable filesystem snapshot feature for the test regardless of the default conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion()); + conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, + HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, -1); conf.setTimeDuration(OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL, 100, TimeUnit.MILLISECONDS); @@ -324,23 +327,10 @@ private static void assertFinalizationException(OMException omException) { * (status FINALIZATION_DONE). */ private void finalizeOMUpgrade() throws Exception { - // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); - UpgradeFinalization.StatusAndMessages finalizationResponse = - omClient.finalizeUpgrade(upgradeClientID); - - // The status should transition as soon as the client call above returns - assertTrue(isStarting(finalizationResponse.status())); - // Wait for the finalization to be marked as done. - // 10s timeout should be plenty. - await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> { - final UpgradeFinalization.StatusAndMessages progress = - omClient.queryUpgradeFinalizationProgress( - upgradeClientID, false, false); - return isDone(progress.status()); - }); + cluster.getStorageContainerLocationClient().finalizeUpgrade(); + OMUpgradeTestUtils.waitForFinalization(omClient); } @AfterAll diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f89dae3b3c0a..b17eed0bbe35 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -300,6 +300,7 @@ import org.apache.hadoop.ozone.om.service.QuotaRepairTask; import org.apache.hadoop.ozone.om.snapshot.defrag.SnapshotDefragService; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizeService; import org.apache.hadoop.ozone.om.upgrade.OMVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OzoneManagerAdminService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -463,6 +464,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private KeyProviderCryptoExtension kmsProvider; private final OMVersionManager versionManager; + private OMUpgradeFinalizeService omUpgradeFinalizeService; private final ReplicationConfigValidator replicationConfigValidator; @@ -639,6 +641,12 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) this.ozoneLockProvider = new OzoneLockProvider(getKeyPathLockEnabled(), getEnableFileSystemPaths()); + if (versionManager.needsFinalization()) { + long intervalMs = conf.getTimeDuration(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, + OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + omUpgradeFinalizeService = new OMUpgradeFinalizeService(this, versionManager, getScmClient(), intervalMs); + } + // For testing purpose only, not hit scm from om as Hadoop UGI can't login // two principals in the same JVM. ScmInfo scmInfo; @@ -1900,6 +1908,10 @@ public void start() throws IOException { bootstrap(omNodeDetails); } + if (omUpgradeFinalizeService != null) { + omUpgradeFinalizeService.start(); + } + omState = State.RUNNING; auditMap.put("NewOmState", omState.name()); SYSTEMAUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMSystemAction.STARTUP, auditMap)); @@ -2441,6 +2453,10 @@ public boolean stop() { bucketUtilizationMetrics.unRegister(); } + if (omUpgradeFinalizeService != null) { + omUpgradeFinalizeService.shutdown(); + } + if (versionManager != null) { versionManager.close(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java new file mode 100644 index 000000000000..bb41a3c6b277 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ScmClient; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.ratis.protocol.ClientId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A background service that periodically checks whether SCM has completed finalization of an upgrade and, if so, + * finalizes the OM upgrade. + */ +public class OMUpgradeFinalizeService extends BackgroundService { + + private static final Logger LOG = LoggerFactory.getLogger(OMUpgradeFinalizeService.class); + + private static final int THREAD_POOL_SIZE = 1; + private static final TimeUnit INTERVAL_UNIT = TimeUnit.MILLISECONDS; + private static final long TIMEOUT = 60000; + private static final AtomicLong RUN_COUNT = new AtomicLong(0); + + private final OzoneManager ozoneManager; + private final OMVersionManager versionManager; + private final ScmClient scmClient; + + /** + * Creates an {@code OMUpgradeFinalizeService} with a custom check interval. + * Primarily intended for testing. + * + * @param ozoneManager the OzoneManager instance + * @param versionManager the {@link OMVersionManager} to query the finalization status + * @param scmClient the scmClient instance used to query SCM + * @param intervalMs the duration to wait between checks + */ + public OMUpgradeFinalizeService(OzoneManager ozoneManager, OMVersionManager versionManager, ScmClient scmClient, + long intervalMs) { + super("OMUpgradeFinalizeService", intervalMs, INTERVAL_UNIT, THREAD_POOL_SIZE, TIMEOUT, + ozoneManager.getThreadNamePrefix()); + this.ozoneManager = ozoneManager; + this.versionManager = versionManager; + this.scmClient = scmClient; + } + + @Override + public BackgroundTaskQueue getTasks() { + BackgroundTaskQueue queue = new BackgroundTaskQueue(); + if (ozoneManager.isLeaderReady() && versionManager.needsFinalization()) { + queue.add(new UpgradeStatusCheckTask()); + } + return queue; + } + + /** + * Periodic task that checks upgrade finalization status and logs the result. + */ + private class UpgradeStatusCheckTask implements BackgroundTask { + + @Override + public BackgroundTaskResult call() { + if (!ozoneManager.isLeaderReady()) { + LOG.debug("OMUpgradeFinalizeService: skipping check — not the leader."); + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + if (versionManager.needsFinalization()) { + try { + if (scmClient == null) { + LOG.info("SCM client is null"); + } + if (scmClient.getContainerClient() == null) { + LOG.info("SCM client's container client is null"); + } + HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); + LOG.info("+++ polled scm and the status is {}", upgradeStatus.getShouldFinalize()); + if (upgradeStatus.getShouldFinalize()) { + LOG.info("The SCM Upgrade has been finalized. OM will now finalize"); + + OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) + .setClientId("todo") + .build(); + OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager); + omRequest = omClientRequest.preExecute(ozoneManager); + OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, ClientId.randomId(), + RUN_COUNT.getAndIncrement()); + } + } catch (Exception e) { + LOG.error("An exception occurred while trying to check the SCM Upgrade status or finalize OM", e); + } + } + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java new file mode 100644 index 000000000000..6b7303908f32 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ScmClient; +import org.apache.hadoop.ozone.upgrade.UpgradeException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link OMUpgradeFinalizeService}. + * Uses {@link org.apache.hadoop.hdds.utils.BackgroundService#runPeriodicalTaskNow()} to execute + * tasks synchronously on the test thread, avoiding timing-dependent polling. + */ +public class TestOMUpgradeFinalizeService { + + // A long interval so the scheduler never fires automatically during tests; + // we drive execution manually via runPeriodicalTaskNow(). + private static final long INTERVAL_MS = 60_000; + + private OzoneManager ozoneManager; + private OMVersionManager versionManager; + private ScmClient scmClient; + private StorageContainerLocationProtocol containerClient; + private OMUpgradeFinalizeService service; + + @BeforeEach + void setUp() { + ozoneManager = mock(OzoneManager.class); + when(ozoneManager.getThreadNamePrefix()).thenReturn(""); + + versionManager = mock(OMVersionManager.class); + + containerClient = mock(StorageContainerLocationProtocol.class); + scmClient = mock(ScmClient.class); + when(scmClient.getContainerClient()).thenReturn(containerClient); + + service = new OMUpgradeFinalizeService(ozoneManager, versionManager, scmClient, INTERVAL_MS); + } + + /** + * When the OM is not the leader, getTasks() should return an empty queue + * and no interaction with the version manager or SCM client should occur. + */ + @Test + void testNoTasksSubmittedWhenNotLeader() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(false); + when(versionManager.needsFinalization()).thenReturn(true); + + service.runPeriodicalTaskNow(); + + verify(scmClient, never()).getContainerClient(); + verify(versionManager, never()).finalizeUpgrade(); + } + + /** + * When finalization is not needed, getTasks() should return an empty queue + * and finalizeUpgrade() should never be called. + */ + @Test + void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(false); + + service.runPeriodicalTaskNow(); + + verify(scmClient, never()).getContainerClient(); + verify(versionManager, never()).finalizeUpgrade(); + } + + /** + * When the OM is the leader, finalization is needed, and SCM reports + * shouldFinalize=true, finalizeUpgrade() should be called exactly once. + */ + @Test + void testFinalizationTriggeredWhenScmIsFinalized() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + + HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(true) + .setShouldFinalize(true) + .setNumDatanodesFinalized(3) + .setNumDatanodesTotal(3) + .build(); + when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); + + service.runPeriodicalTaskNow(); + + verify(scmClient).getContainerClient(); + verify(containerClient).queryUpgradeStatus(); + verify(versionManager, times(1)).finalizeUpgrade(); + } + + /** + * When SCM reports shouldFinalize=false (SCM is not yet finalized), + * the OM should not attempt to finalize. + */ + @Test + void testFinalizationSkippedWhenScmNotYetFinalized() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + + HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(false) + .setShouldFinalize(false) + .setNumDatanodesFinalized(0) + .setNumDatanodesTotal(3) + .build(); + when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); + + service.runPeriodicalTaskNow(); + + verify(containerClient).queryUpgradeStatus(); + verify(versionManager, never()).finalizeUpgrade(); + } + + /** + * When the SCM client throws an IOException, the service should absorb it + * and not call finalizeUpgrade(). + */ + @Test + void testExceptionFromScmClientIsHandledGracefully() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + when(containerClient.queryUpgradeStatus()).thenThrow(new IOException("SCM unavailable")); + + // runPeriodicalTaskNow() calls call() directly; the catch block in the task + // swallows the exception, so this should complete without throwing. + service.runPeriodicalTaskNow(); + + verify(containerClient).queryUpgradeStatus(); + verify(versionManager, never()).finalizeUpgrade(); + } + + /** + * When finalizeUpgrade() itself throws, the service should absorb the error. + */ + @Test + void testExceptionFromFinalizeUpgradeIsHandledGracefully() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + + HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(true) + .setShouldFinalize(true) + .setNumDatanodesFinalized(3) + .setNumDatanodesTotal(3) + .build(); + when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); + UpgradeException upgradeException = new UpgradeException("finalization failed", + UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED); + org.mockito.Mockito.doThrow(upgradeException).when(versionManager).finalizeUpgrade(); + + // The catch block in the task swallows the exception. + service.runPeriodicalTaskNow(); + + verify(versionManager, atLeastOnce()).finalizeUpgrade(); + } +} From 6d854579c09d0a2ddc733f4746d7c5f03123e140 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 May 2026 11:52:11 +0100 Subject: [PATCH 02/37] Remove log messages --- .../hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index bb41a3c6b277..71396aa1bc23 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -90,14 +90,7 @@ public BackgroundTaskResult call() { } if (versionManager.needsFinalization()) { try { - if (scmClient == null) { - LOG.info("SCM client is null"); - } - if (scmClient.getContainerClient() == null) { - LOG.info("SCM client's container client is null"); - } HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); - LOG.info("+++ polled scm and the status is {}", upgradeStatus.getShouldFinalize()); if (upgradeStatus.getShouldFinalize()) { LOG.info("The SCM Upgrade has been finalized. OM will now finalize"); From 136574b795b8ea4fc19291cb6530895ca4be9d47 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 May 2026 12:28:25 +0100 Subject: [PATCH 03/37] Fix findbugs --- .../om/upgrade/TestOMUpgradeFinalizeService.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 6b7303908f32..68ca3270e370 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.io.IOException; @@ -75,7 +76,7 @@ void testNoTasksSubmittedWhenNotLeader() throws Exception { service.runPeriodicalTaskNow(); - verify(scmClient, never()).getContainerClient(); + verifyNoInteractions(scmClient); verify(versionManager, never()).finalizeUpgrade(); } @@ -90,7 +91,7 @@ void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception { service.runPeriodicalTaskNow(); - verify(scmClient, never()).getContainerClient(); + verifyNoInteractions(scmClient); verify(versionManager, never()).finalizeUpgrade(); } @@ -113,9 +114,8 @@ void testFinalizationTriggeredWhenScmIsFinalized() throws Exception { service.runPeriodicalTaskNow(); - verify(scmClient).getContainerClient(); verify(containerClient).queryUpgradeStatus(); - verify(versionManager, times(1)).finalizeUpgrade(); + // TODO - need to validate the ratis call } /** @@ -180,7 +180,6 @@ void testExceptionFromFinalizeUpgradeIsHandledGracefully() throws Exception { // The catch block in the task swallows the exception. service.runPeriodicalTaskNow(); - - verify(versionManager, atLeastOnce()).finalizeUpgrade(); + // TODO - these tests need a bit of rework since moving to ratis. } } From 8847638cf5303d9b7be0ce0246d284cb705127cf Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 May 2026 16:13:03 +0100 Subject: [PATCH 04/37] Fix checkstyle --- .../hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 68ca3270e370..34287778b5e6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -17,10 +17,8 @@ package org.apache.hadoop.ozone.om.upgrade; -import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; From 658121879f9abcd063867692c02b8a805662daec Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 May 2026 18:17:56 +0100 Subject: [PATCH 05/37] Fix config check and update unit tests to verify ratis interactions --- .../src/main/resources/ozone-default.xml | 6 ++ .../upgrade/TestOMUpgradeFinalizeService.java | 56 ++++++++++++------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index aeda046dcd74..2c90c5440a7e 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -5028,4 +5028,10 @@ OZONE, RATIS, OM The maximum number of events that can be pending in OM Ratis. + + ozone.om.upgrade.finalization.check.interval + 1m + OM + If OM is unfinalized, how frequently it should poll SCM to trigger finalization + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 34287778b5e6..69f289e75627 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -17,18 +17,22 @@ package org.apache.hadoop.ozone.om.upgrade; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.io.IOException; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import com.google.protobuf.ServiceException; +import org.apache.ratis.protocol.ClientId; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmClient; -import org.apache.hadoop.ozone.upgrade.UpgradeException; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -47,14 +51,22 @@ public class TestOMUpgradeFinalizeService { private OMVersionManager versionManager; private ScmClient scmClient; private StorageContainerLocationProtocol containerClient; + private OzoneManagerRatisServer omRatisServer; private OMUpgradeFinalizeService service; @BeforeEach - void setUp() { + void setUp() throws Exception { ozoneManager = mock(OzoneManager.class); when(ozoneManager.getThreadNamePrefix()).thenReturn(""); versionManager = mock(OMVersionManager.class); + // preExecute() calls ozoneManager.getVersionManager().getApparentVersion().serialize() + when(ozoneManager.getVersionManager()).thenReturn(versionManager); + when(versionManager.getApparentVersion()).thenReturn(OzoneManagerVersion.DEFAULT_VERSION); + + omRatisServer = mock(OzoneManagerRatisServer.class); + // OzoneManagerRatisUtils.submitRequest() calls ozoneManager.getOmRatisServer().submitRequest(...) + when(ozoneManager.getOmRatisServer()).thenReturn(omRatisServer); containerClient = mock(StorageContainerLocationProtocol.class); scmClient = mock(ScmClient.class); @@ -65,7 +77,7 @@ void setUp() { /** * When the OM is not the leader, getTasks() should return an empty queue - * and no interaction with the version manager or SCM client should occur. + * and no interaction with the SCM client or Ratis server should occur. */ @Test void testNoTasksSubmittedWhenNotLeader() throws Exception { @@ -75,12 +87,12 @@ void testNoTasksSubmittedWhenNotLeader() throws Exception { service.runPeriodicalTaskNow(); verifyNoInteractions(scmClient); - verify(versionManager, never()).finalizeUpgrade(); + verifyNoInteractions(omRatisServer); } /** * When finalization is not needed, getTasks() should return an empty queue - * and finalizeUpgrade() should never be called. + * and no SCM query or Ratis submission should occur. */ @Test void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception { @@ -90,12 +102,12 @@ void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception { service.runPeriodicalTaskNow(); verifyNoInteractions(scmClient); - verify(versionManager, never()).finalizeUpgrade(); + verifyNoInteractions(omRatisServer); } /** * When the OM is the leader, finalization is needed, and SCM reports - * shouldFinalize=true, finalizeUpgrade() should be called exactly once. + * shouldFinalize=true, a FinalizeUpgrade request should be submitted via Ratis. */ @Test void testFinalizationTriggeredWhenScmIsFinalized() throws Exception { @@ -113,12 +125,13 @@ void testFinalizationTriggeredWhenScmIsFinalized() throws Exception { service.runPeriodicalTaskNow(); verify(containerClient).queryUpgradeStatus(); - // TODO - need to validate the ratis call + // Implementation submits a FinalizeUpgrade request through Ratis + verify(omRatisServer).submitRequest(any(), any(ClientId.class), anyLong()); } /** * When SCM reports shouldFinalize=false (SCM is not yet finalized), - * the OM should not attempt to finalize. + * no Ratis request should be submitted. */ @Test void testFinalizationSkippedWhenScmNotYetFinalized() throws Exception { @@ -136,12 +149,12 @@ void testFinalizationSkippedWhenScmNotYetFinalized() throws Exception { service.runPeriodicalTaskNow(); verify(containerClient).queryUpgradeStatus(); - verify(versionManager, never()).finalizeUpgrade(); + verifyNoInteractions(omRatisServer); } /** * When the SCM client throws an IOException, the service should absorb it - * and not call finalizeUpgrade(). + * and not submit any Ratis request. */ @Test void testExceptionFromScmClientIsHandledGracefully() throws Exception { @@ -149,19 +162,19 @@ void testExceptionFromScmClientIsHandledGracefully() throws Exception { when(versionManager.needsFinalization()).thenReturn(true); when(containerClient.queryUpgradeStatus()).thenThrow(new IOException("SCM unavailable")); - // runPeriodicalTaskNow() calls call() directly; the catch block in the task - // swallows the exception, so this should complete without throwing. + // The catch block in the task swallows the exception. service.runPeriodicalTaskNow(); verify(containerClient).queryUpgradeStatus(); - verify(versionManager, never()).finalizeUpgrade(); + verifyNoInteractions(omRatisServer); } /** - * When finalizeUpgrade() itself throws, the service should absorb the error. + * When the Ratis submission throws, the service should absorb the error + * and not propagate the exception. */ @Test - void testExceptionFromFinalizeUpgradeIsHandledGracefully() throws Exception { + void testExceptionFromRatisSubmitIsHandledGracefully() throws Exception { when(ozoneManager.isLeaderReady()).thenReturn(true); when(versionManager.needsFinalization()).thenReturn(true); @@ -172,12 +185,13 @@ void testExceptionFromFinalizeUpgradeIsHandledGracefully() throws Exception { .setNumDatanodesTotal(3) .build(); when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); - UpgradeException upgradeException = new UpgradeException("finalization failed", - UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED); - org.mockito.Mockito.doThrow(upgradeException).when(versionManager).finalizeUpgrade(); + when(omRatisServer.submitRequest(any(), any(ClientId.class), anyLong())) + .thenThrow(new ServiceException("Ratis unavailable")); // The catch block in the task swallows the exception. service.runPeriodicalTaskNow(); - // TODO - these tests need a bit of rework since moving to ratis. + + // submitRequest was attempted but threw — service should not propagate the exception. + verify(omRatisServer).submitRequest(any(), any(ClientId.class), anyLong()); } } From 26264465588b55810c5a1048ad671de2004c1635 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 8 May 2026 20:53:44 +0100 Subject: [PATCH 06/37] Fix style --- .../hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 69f289e75627..d686110b1d35 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -24,15 +24,15 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import com.google.protobuf.ServiceException; import java.io.IOException; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import com.google.protobuf.ServiceException; -import org.apache.ratis.protocol.ClientId; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmClient; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.ratis.protocol.ClientId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; From 5823be4174ac2ab85edcab50982ea05968db89f4 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 11 May 2026 12:49:18 +0100 Subject: [PATCH 07/37] Stop background service thread after finalization no longer needed --- .../om/upgrade/OMUpgradeFinalizeService.java | 16 ++++++- .../upgrade/TestOMUpgradeFinalizeService.java | 47 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index 71396aa1bc23..5d8a0110472f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.upgrade; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.BackgroundService; @@ -49,6 +50,7 @@ public class OMUpgradeFinalizeService extends BackgroundService { private final OzoneManager ozoneManager; private final OMVersionManager versionManager; private final ScmClient scmClient; + private final AtomicBoolean stopInitiated = new AtomicBoolean(false); /** * Creates an {@code OMUpgradeFinalizeService} with a custom check interval. @@ -71,7 +73,19 @@ public OMUpgradeFinalizeService(OzoneManager ozoneManager, OMVersionManager vers @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); - if (ozoneManager.isLeaderReady() && versionManager.needsFinalization()) { + if (!versionManager.needsFinalization()) { + // Finalization is done (or was never needed), so this service can now shutdown. To avoid deadlocking on the + // executor.awaitTermination by calling shutdown directly, spawn a thread to perform the shutdown which will + // block until this task / thread completes in the executor. + if (stopInitiated.compareAndSet(false, true)) { + LOG.info("OMUpgradeFinalizeService: finalization is no longer needed, shutting down."); + Thread stopper = new Thread(this::shutdown, "OMUpgradeFinalizeService-stopper"); + stopper.setDaemon(true); + stopper.start(); + } + return queue; // empty — PeriodicalTask.run() will return without scheduling work + } + if (ozoneManager.isLeaderReady()) { queue.add(new UpgradeStatusCheckTask()); } return queue; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index d686110b1d35..1b92dcfefb5f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -17,6 +17,8 @@ package org.apache.hadoop.ozone.om.upgrade; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; @@ -26,6 +28,9 @@ import com.google.protobuf.ServiceException; import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.ozone.OzoneManagerVersion; @@ -169,6 +174,48 @@ void testExceptionFromScmClientIsHandledGracefully() throws Exception { verifyNoInteractions(omRatisServer); } + /** + * When {@code needsFinalization()} returns {@code false}, {@link OMUpgradeFinalizeService#getTasks()} + * should spawn a stopper thread that calls {@link OMUpgradeFinalizeService#shutdown()} exactly once, + * even when {@code getTasks()} is driven multiple times (guarded by the internal + * {@code stopInitiated} AtomicBoolean). No SCM or Ratis interactions should occur. + *

+ * The service is subclassed to intercept {@code shutdown()} via a {@link CountDownLatch}, + * avoiding both actual executor teardown and any need for {@code Thread.sleep}. + */ + @Test + void testShutdownTriggeredExactlyOnceWhenFinalizationNoLongerNeeded() throws Exception { + AtomicInteger shutdownCount = new AtomicInteger(0); + CountDownLatch firstShutdown = new CountDownLatch(1); + + OMUpgradeFinalizeService testService = new OMUpgradeFinalizeService( + ozoneManager, versionManager, scmClient, INTERVAL_MS) { + @Override + public synchronized void shutdown() { + shutdownCount.incrementAndGet(); + firstShutdown.countDown(); + // Don't propagate to super — avoids racing on the test executor. + } + }; + + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(false); + + // Drive getTasks() three times; the stopper thread should only fire once. + testService.runPeriodicalTaskNow(); + testService.runPeriodicalTaskNow(); + testService.runPeriodicalTaskNow(); + + // Wait for the single stopper thread to call shutdown(). + assertTrue(firstShutdown.await(5, TimeUnit.SECONDS), + "shutdown() should have been called by the stopper thread within 5 s"); + // incrementAndGet() happens before countDown(), so the count is already final — no sleep needed. + assertEquals(1, shutdownCount.get(), + "shutdown() must be called exactly once despite multiple getTasks() invocations"); + verifyNoInteractions(scmClient); + verifyNoInteractions(omRatisServer); + } + /** * When the Ratis submission throws, the service should absorb the error * and not propagate the exception. From cf7bbb2d1a68286904184bccf3907322c8bb2da6 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 11 May 2026 12:54:49 +0100 Subject: [PATCH 08/37] Fix typo in ozone-default --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 2c90c5440a7e..dc16b29b77a5 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -5029,7 +5029,7 @@ The maximum number of events that can be pending in OM Ratis. - ozone.om.upgrade.finalization.check.interval + ozone.om.upgrade.finalization.check.interval 1m OM If OM is unfinalized, how frequently it should poll SCM to trigger finalization From 77fbc35ff987873784b1fd0a1bfd6ec192e07140 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 12 May 2026 11:26:30 +0100 Subject: [PATCH 09/37] Address review comments --- .../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 4 ++-- .../ozone/om/TestOMUpgradeFinalization.java | 2 +- .../om/upgrade/OMUpgradeFinalizeService.java | 15 +++++++-------- .../om/upgrade/TestOMUpgradeFinalizeService.java | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java index e563cba60528..415a8de7d8a8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java @@ -50,7 +50,6 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -64,6 +63,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -112,7 +112,7 @@ public void init() throws Exception { conf.setTimeDuration(OZONE_OM_LEASE_HARD_LIMIT, EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s"); - conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OzoneManagerVersion.ATOMIC_REWRITE_KEY.serialize()); + conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion()); conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.layoutVersion()); conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index 3c04493d213b..a18ae113ce01 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -103,9 +103,9 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_CANCEL, AuditEventStatus.SUCCESS); // Send the finalize command to SCM which triggers the OM finalize when SCM reports it is complete. cluster.getStorageContainerLocationClient().finalizeUpgrade(); - AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); waitForFinalization(omClient); + AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); cluster.restartOzoneManager(downedOM, true); OzoneManagerStateMachine omStateMachine = downedOM.getOmRatisServer() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index 5d8a0110472f..3587f62a1d01 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -28,7 +28,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmClient; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; -import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.ratis.protocol.ClientId; import org.slf4j.Logger; @@ -54,7 +53,6 @@ public class OMUpgradeFinalizeService extends BackgroundService { /** * Creates an {@code OMUpgradeFinalizeService} with a custom check interval. - * Primarily intended for testing. * * @param ozoneManager the OzoneManager instance * @param versionManager the {@link OMVersionManager} to query the finalization status @@ -62,7 +60,7 @@ public class OMUpgradeFinalizeService extends BackgroundService { * @param intervalMs the duration to wait between checks */ public OMUpgradeFinalizeService(OzoneManager ozoneManager, OMVersionManager versionManager, ScmClient scmClient, - long intervalMs) { + long intervalMs) { super("OMUpgradeFinalizeService", intervalMs, INTERVAL_UNIT, THREAD_POOL_SIZE, TIMEOUT, ozoneManager.getThreadNamePrefix()); this.ozoneManager = ozoneManager; @@ -110,12 +108,13 @@ public BackgroundTaskResult call() { OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) - .setClientId("todo") + .setClientId(ozoneManager.getOMNodeId()) .build(); - OMClientRequest omClientRequest = OzoneManagerRatisUtils.createClientRequest(omRequest, ozoneManager); - omRequest = omClientRequest.preExecute(ozoneManager); - OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, ClientId.randomId(), - RUN_COUNT.getAndIncrement()); + OzoneManagerProtocolProtos.OMResponse response = OzoneManagerRatisUtils.submitRequest( + ozoneManager, omRequest, ClientId.randomId(), RUN_COUNT.getAndIncrement()); + if (!response.getSuccess()) { + LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis. {}", response.getMessage()); + } } } catch (Exception e) { LOG.error("An exception occurred while trying to check the SCM Upgrade status or finalize OM", e); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 1b92dcfefb5f..244b9b1ef8b1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -60,7 +60,7 @@ public class TestOMUpgradeFinalizeService { private OMUpgradeFinalizeService service; @BeforeEach - void setUp() throws Exception { + void setUp() { ozoneManager = mock(OzoneManager.class); when(ozoneManager.getThreadNamePrefix()).thenReturn(""); From e042fad5ce08067d5f5cfdf23dec90a0ca48447c Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 12 May 2026 15:58:10 +0100 Subject: [PATCH 10/37] Fix failing test --- .../hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 244b9b1ef8b1..61022e90e694 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -63,6 +63,7 @@ public class TestOMUpgradeFinalizeService { void setUp() { ozoneManager = mock(OzoneManager.class); when(ozoneManager.getThreadNamePrefix()).thenReturn(""); + when(ozoneManager.getOMNodeId()).thenReturn("clientId"); versionManager = mock(OMVersionManager.class); // preExecute() calls ozoneManager.getVersionManager().getApparentVersion().serialize() From 7dad7f16e803b2621e180843f801740a674a654c Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 12 May 2026 15:58:45 +0100 Subject: [PATCH 11/37] Finalization should goto system audit log as its happening in the background --- .../hadoop/ozone/audit/AuditLogTestUtils.java | 18 ++++++++++++++- .../ozone/om/TestOMUpgradeFinalization.java | 2 +- .../src/test/resources/auditlog.properties | 23 ++++++++++++------- .../upgrade/OMFinalizeUpgradeRequest.java | 2 +- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java index 0b05d4d01f0e..3020f4000a55 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java @@ -42,6 +42,7 @@ private AuditLogTestUtils() { */ public static void enableAuditLog() { System.setProperty("log4j.configurationFile", "auditlog.properties"); + System.setProperty("log4j.configurationFile", "auditlog.properties"); } /** @@ -55,8 +56,22 @@ public static void verifyAuditLog(AuditAction action, 1000, 10000); } + /** + * Searches for the given action in the system audit log file. + */ + public static void verifySystemAuditLog(AuditAction action, + AuditEventStatus eventStatus) throws InterruptedException, TimeoutException { + waitFor( + () -> fileContains(SYSTEM_AUDITLOG_FILENAME, action.getAction(), eventStatus.getStatus()), + 1000, 10000); + } + public static boolean auditLogContains(String... strings) { - File file = new File(AUDITLOG_FILENAME); + return fileContains(AUDITLOG_FILENAME, strings); + } + + private static boolean fileContains(String filename, String... strings) { + File file = new File(filename); try { String contents = FileUtils.readFileToString(file, UTF_8); for (String s : strings) { @@ -72,6 +87,7 @@ public static boolean auditLogContains(String... strings) { public static void truncateAuditLogFile() throws IOException { Files.write(Paths.get(AUDITLOG_FILENAME), new byte[0]); + Files.write(Paths.get(SYSTEM_AUDITLOG_FILENAME), new byte[0]); } public static void deleteAuditLogFile() { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index a18ae113ce01..243abe3647f5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -105,7 +105,7 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { cluster.getStorageContainerLocationClient().finalizeUpgrade(); waitForFinalization(omClient); - AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); + AuditLogTestUtils.verifySystemAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); cluster.restartOzoneManager(downedOM, true); OzoneManagerStateMachine omStateMachine = downedOM.getOmRatisServer() diff --git a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties index c5e9f8d2c7dc..bc5e6cd2bd00 100644 --- a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties +++ b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties @@ -52,27 +52,34 @@ filter.write.onMismatch = NEUTRAL # TRACE (least specific, a lot of data) # ALL (least specific, all data) -appenders = console, audit +appenders = console, audit, systemaudit appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +appender.console.layout.pattern = %-5level | %c{1} | %msg%n appender.audit.type = File appender.audit.name = AUDITLOG appender.audit.fileName=audit.log appender.audit.layout.type=PatternLayout -appender.audit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +appender.audit.layout.pattern= %-5level | %c{1} | %C | %msg%n -loggers=audit,omSystemAudit +loggers=audit, systemaudit logger.audit.name=OMAudit logger.audit.level = INFO logger.audit.appenderRefs = audit logger.audit.appenderRef.file.ref = AUDITLOG -logger.omSystemAudit.name=OMSystemAudit -logger.omSystemAudit.level = INFO -logger.omSystemAudit.appenderRefs = audit -logger.omSystemAudit.appenderRef.file.ref = AUDITLOG + +appender.systemaudit.type = File +appender.systemaudit.name = SYSTEMAUDITLOG +appender.systemaudit.fileName=system_audit.log +appender.systemaudit.layout.type=PatternLayout +appender.systemaudit.layout.pattern= %-5level | %c{1} | %C | %msg%n + +logger.systemaudit.name=OMSystemAudit +logger.systemaudit.level = INFO +logger.systemaudit.appenderRefs = audit +logger.systemaudit.appenderRef.file.ref = SYSTEMAUDITLOG rootLogger.level = INFO rootLogger.appenderRefs = stdout diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java index 2bff57e9a8dc..6c4291b176fe 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java @@ -59,7 +59,7 @@ public OMFinalizeUpgradeRequest(OMRequest omRequest) { @Override public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) { LOG.trace("Request: {}", getOmRequest()); - AuditLogger auditLogger = ozoneManager.getAuditLogger(); + AuditLogger auditLogger = ozoneManager.getSystemAuditLogger(); OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); OMResponse.Builder responseBuilder = OmResponseUtil.getOMResponseBuilder(getOmRequest()); From ce82754105f11355b2c5e6dcfd5f55ae7ef1fe52 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 12 May 2026 16:15:34 +0100 Subject: [PATCH 12/37] Fix compile issue --- .../java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java index 3020f4000a55..e6912d36c820 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java @@ -32,6 +32,7 @@ */ public final class AuditLogTestUtils { private static final String AUDITLOG_FILENAME = "audit.log"; + private static final String SYSTEM_AUDITLOG_FILENAME = "system_audit.log"; private AuditLogTestUtils() { } From 8e6ddbec67850732ace75c704e05a3c4d2250641 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 15 May 2026 16:44:10 +0100 Subject: [PATCH 13/37] Add a meta-table key that must be present for finalization of OM to happen --- .../org/apache/hadoop/ozone/OzoneConsts.java | 3 +++ .../hadoop/fs/ozone/TestHSyncUpgrade.java | 7 +++--- .../ozone/om/TestOMBucketLayoutUpgrade.java | 6 +++-- .../ozone/om/TestOMUpgradeFinalization.java | 9 ++++---- .../om/multitenant/TestMultiTenantVolume.java | 5 +++-- .../ozone/om/snapshot/TestOmSnapshot.java | 6 ++--- .../om/upgrade/OMUpgradeFinalizeService.java | 14 ++++++++++++ .../upgrade/TestOMUpgradeFinalizeService.java | 22 +++++++++++++++++-- 8 files changed, 55 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 5d0b826c17e1..1aa5961eb368 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -439,6 +439,9 @@ public final class OzoneConsts { // Apparent Version written into Meta Table ONLY during finalization. // The name "layout version" is kept for backwards compatibility. public static final String APPARENT_VERSION_KEY = "#LAYOUTVERSION"; + // Key written into the Meta table when finalization is needed and a finalization command has been received + // to trigger the process + public static final String FINALIZATION_IN_PROGRESS_KEY = "#FINALIZATION_IN_PROGRESS"; // Kerberos constants public static final String KERBEROS_CONFIG_VALUE = "kerberos"; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java index 415a8de7d8a8..b00177ec0054 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java @@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -113,8 +114,6 @@ public void init() throws Exception { EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s"); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion()); - conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, - HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.layoutVersion()); conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); ClientConfigForTesting.newBuilder(StorageUnit.BYTES) @@ -215,7 +214,9 @@ private void finalizeOMUpgrade() throws Exception { // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - cluster.getStorageContainerLocationClient().finalizeUpgrade(); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); OMUpgradeTestUtils.waitForFinalization(omClient); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index fe5a7629a01e..0444f94608d3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -90,7 +91,6 @@ class TestOMBucketLayoutUpgrade { void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, fromVersion.serialize()); - conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); String omServiceId = UUID.randomUUID().toString(); MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf); @@ -155,7 +155,9 @@ void allowsLegacyBucketBeforeUpgrade() throws Exception { @Test @Order(DURING_UPGRADE) void finalizeUpgrade() throws Exception { - cluster.getStorageContainerLocationClient().finalizeUpgrade(); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); waitForFinalization(omClient); final String expectedVersion = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index 243abe3647f5..445f845aa075 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogTestUtils; @@ -71,9 +72,6 @@ public static void shutdown() { void testOMUpgradeFinalizationWithOneOMDown() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, INITIAL_VERSION.layoutVersion()); - // Finalization is triggered when SCM is finalized, so we first start it unfinalized and then finalize it to trigger - // the OM finalization. - conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); try (MiniOzoneHAClusterImpl cluster = newCluster(conf)) { cluster.waitForClusterToBeReady(); @@ -101,8 +99,9 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_PREPARE, AuditEventStatus.SUCCESS); omClient.cancelOzoneManagerPrepare(); AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_CANCEL, AuditEventStatus.SUCCESS); - // Send the finalize command to SCM which triggers the OM finalize when SCM reports it is complete. - cluster.getStorageContainerLocationClient().finalizeUpgrade(); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); waitForFinalization(omClient); AuditLogTestUtils.verifySystemAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index 89d3823b93e9..4f8c0934305c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@ -75,7 +75,6 @@ public static void initClusterProvider() throws Exception { OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true); conf.setBoolean(OZONE_OM_MULTITENANCY_ENABLED, true); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.INITIAL_VERSION.layoutVersion()); - conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion()); conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf) .withoutDatanodes(); @@ -146,7 +145,9 @@ private static void finalizeOMUpgrade() // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - cluster.getStorageContainerLocationClient().finalizeUpgrade(); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); OMUpgradeTestUtils.waitForFinalization(omClient); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 36547377714c..83ee33dd1c6a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -234,8 +234,6 @@ private void init() throws Exception { // Enable filesystem snapshot feature for the test regardless of the default conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion()); - conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, - HDDSLayoutFeature.HADOOP_PRC_PORTS_IN_DATANODEDETAILS.layoutVersion()); conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, -1); @@ -329,7 +327,9 @@ private static void assertFinalizationException(OMException omException) { private void finalizeOMUpgrade() throws Exception { final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - cluster.getStorageContainerLocationClient().finalizeUpgrade(); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); OMUpgradeTestUtils.waitForFinalization(omClient); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index 3587f62a1d01..312740c660d7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -17,6 +17,8 @@ package org.apache.hadoop.ozone.om.upgrade; +import static org.apache.hadoop.ozone.OzoneConsts.FINALIZATION_IN_PROGRESS_KEY; + import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -25,6 +27,8 @@ import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmClient; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -45,6 +49,7 @@ public class OMUpgradeFinalizeService extends BackgroundService { private static final TimeUnit INTERVAL_UNIT = TimeUnit.MILLISECONDS; private static final long TIMEOUT = 60000; private static final AtomicLong RUN_COUNT = new AtomicLong(0); + private static final CacheKey FINALIZATION_CACHE_KEY = new CacheKey<>(FINALIZATION_IN_PROGRESS_KEY); private final OzoneManager ozoneManager; private final OMVersionManager versionManager; @@ -102,6 +107,15 @@ public BackgroundTaskResult call() { } if (versionManager.needsFinalization()) { try { + // To finalize OM, first finalization needs to have been started. Then SCM needs to indicate that it has + // completed its finalization work. Only once both of those things have happened can OM finalize. + CacheValue finalizationValue = + ozoneManager.getMetadataManager().getMetaTable().getCacheValue(FINALIZATION_CACHE_KEY); + if (finalizationValue.getCacheValue() == null) { + LOG.debug("OMUpgradeFinalizeService: skipping check — finalization is not in progress."); + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); if (upgradeStatus.getShouldFinalize()) { LOG.info("The SCM Upgrade has been finalized. OM will now finalize"); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 61022e90e694..590bfbb00c5f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -33,7 +33,10 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.utils.db.TypedTable; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmClient; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; @@ -54,6 +57,7 @@ public class TestOMUpgradeFinalizeService { private OzoneManager ozoneManager; private OMVersionManager versionManager; + private TypedTable metaTable; private ScmClient scmClient; private StorageContainerLocationProtocol containerClient; private OzoneManagerRatisServer omRatisServer; @@ -62,8 +66,14 @@ public class TestOMUpgradeFinalizeService { @BeforeEach void setUp() { ozoneManager = mock(OzoneManager.class); + OMMetadataManager metadataManager = mock(OMMetadataManager.class); + metaTable = mock(TypedTable.class); when(ozoneManager.getThreadNamePrefix()).thenReturn(""); when(ozoneManager.getOMNodeId()).thenReturn("clientId"); + when(ozoneManager.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetaTable()).thenReturn(metaTable); + // For most tests, set the finalization command as having been received + when(metaTable.getCacheValue(any())).thenReturn(CacheValue.get(1, "ignored")); versionManager = mock(OMVersionManager.class); // preExecute() calls ozoneManager.getVersionManager().getApparentVersion().serialize() @@ -112,11 +122,11 @@ void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception { } /** - * When the OM is the leader, finalization is needed, and SCM reports + * When the OM is the leader, finalization is needed, the finalization command is given and SCM reports * shouldFinalize=true, a FinalizeUpgrade request should be submitted via Ratis. */ @Test - void testFinalizationTriggeredWhenScmIsFinalized() throws Exception { + void testFinalizationTriggeredWhenScmIsFinalizedAndFinalizationInProgress() throws Exception { when(ozoneManager.isLeaderReady()).thenReturn(true); when(versionManager.needsFinalization()).thenReturn(true); @@ -127,9 +137,17 @@ void testFinalizationTriggeredWhenScmIsFinalized() throws Exception { .setNumDatanodesTotal(3) .build(); when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); + // Finalization command not given yet + when(metaTable.getCacheValue(any())).thenReturn(null); service.runPeriodicalTaskNow(); + verifyNoInteractions(containerClient); + verifyNoInteractions(omRatisServer); + + when(metaTable.getCacheValue(any())).thenReturn(CacheValue.get(1, "ignored")); + service.runPeriodicalTaskNow(); + verify(containerClient).queryUpgradeStatus(); // Implementation submits a FinalizeUpgrade request through Ratis verify(omRatisServer).submitRequest(any(), any(ClientId.class), anyLong()); From 118498b0a96ea92562a40f250a7c40131309e24f Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 15 May 2026 16:58:24 +0100 Subject: [PATCH 14/37] Fix checkstyle --- .../test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 2 -- .../org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 2 -- .../org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java | 2 -- .../hadoop/ozone/om/multitenant/TestMultiTenantVolume.java | 2 -- .../org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java | 2 -- 5 files changed, 10 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java index b00177ec0054..3ef83f944053 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java @@ -41,11 +41,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.storage.BlockInputStream; import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.scm.storage.BufferPool; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index 0444f94608d3..2b25bad6478c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -32,8 +32,6 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index 445f845aa075..54959fa9ea0a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -31,8 +31,6 @@ import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneConsts; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index 4f8c0934305c..c253e8b6e2c5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@ -29,8 +29,6 @@ import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index 83ee33dd1c6a..d7a5bbdef32a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -99,8 +99,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.db.CodecBuffer; import org.apache.hadoop.hdds.utils.db.DBProfile; import org.apache.hadoop.hdds.utils.db.DBStore; From 7e3d80f303e709fef7e48b5fac9653f8a1beeeb4 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 18 May 2026 12:52:42 +0100 Subject: [PATCH 15/37] Fix test failures and meta table setting / deleting --- .../hadoop/ozone/audit/AuditLogTestUtils.java | 5 ++++- .../hadoop/fs/ozone/TestHSyncUpgrade.java | 2 ++ .../ozone/om/TestOMBucketLayoutUpgrade.java | 13 ++++++++++-- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +- .../ozone/om/TestOMUpgradeFinalization.java | 21 +++++++++++++++---- .../om/multitenant/TestMultiTenantVolume.java | 2 ++ .../ozone/om/snapshot/TestOmSnapshot.java | 2 ++ .../upgrade/OMFinalizeUpgradeRequest.java | 4 ++++ .../upgrade/OMFinalizeUpgradeResponse.java | 3 +++ .../om/upgrade/OMUpgradeFinalizeService.java | 9 +++----- 10 files changed, 49 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java index e6912d36c820..81a406d01cfc 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java @@ -43,7 +43,6 @@ private AuditLogTestUtils() { */ public static void enableAuditLog() { System.setProperty("log4j.configurationFile", "auditlog.properties"); - System.setProperty("log4j.configurationFile", "auditlog.properties"); } /** @@ -71,6 +70,10 @@ public static boolean auditLogContains(String... strings) { return fileContains(AUDITLOG_FILENAME, strings); } + public static boolean systemAuditLogContains(String... strings) { + return fileContains(SYSTEM_AUDITLOG_FILENAME, strings); + } + private static boolean fileContains(String filename, String... strings) { File file = new File(filename); try { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java index 3ef83f944053..ff5680a9fd39 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java @@ -215,6 +215,8 @@ private void finalizeOMUpgrade() throws Exception { // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization cluster.getOzoneManager().getMetadataManager().getMetaTable() .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); OMUpgradeTestUtils.waitForFinalization(omClient); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index 2b25bad6478c..3ec9a492accc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -33,6 +33,8 @@ import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneConsts; @@ -154,8 +156,15 @@ void allowsLegacyBucketBeforeUpgrade() throws Exception { @Order(DURING_UPGRADE) void finalizeUpgrade() throws Exception { // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization - cluster.getOzoneManager().getMetadataManager().getMetaTable() - .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManagersList().forEach(om -> { + try { + om.getMetadataManager().getMetaTable().addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + om.getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + } catch (RocksDatabaseException | CodecException e) { + throw new RuntimeException(e); + } + }); waitForFinalization(omClient); final String expectedVersion = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index de2bc98f10c9..cdb40ed36d9e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -297,7 +297,7 @@ public void testInstallSnapshot(@TempDir Path tempDir) throws Exception { String toMatch = String.format( "op=DB_CHECKPOINT_INSTALL {\"leaderId\":\"%s\",\"term\":\"%d\",\"lastAppliedIndex\":\"%d\"}", leaderOMNodeId, leaderOMSnapshotTermIndex, followerOMLastAppliedIndex); - assertTrue(AuditLogTestUtils.auditLogContains(toMatch)); + assertTrue(AuditLogTestUtils.systemAuditLogContains(toMatch)); // Read & Write after snapshot installed. List newKeys = writeKeys(1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index 54959fa9ea0a..e6c84c390d3b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -25,12 +25,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import java.io.IOException; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneConsts; @@ -43,7 +46,6 @@ import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.ratis.util.LifeCycle; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -79,7 +81,7 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { for (OzoneManager om : runningOms) { assertEquals(INITIAL_VERSION, om.getVersionManager().getApparentVersion()); // The OMs have not been finalized yet, so no version has been written to the DB. - Assertions.assertNull(om.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY)); + assertNull(om.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY)); } final int shutdownOMIndex = 2; @@ -98,11 +100,22 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { omClient.cancelOzoneManagerPrepare(); AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_CANCEL, AuditEventStatus.SUCCESS); // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization - cluster.getOzoneManager().getMetadataManager().getMetaTable() - .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManagersList().forEach(om -> { + try { + om.getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + om.getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + } catch (RocksDatabaseException | CodecException e) { + throw new RuntimeException(e); + } + }); waitForFinalization(omClient); AuditLogTestUtils.verifySystemAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); + // Ensure the finalization in progress key has been removed. + assertNull(cluster.getOzoneManager().getMetadataManager() + .getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)); cluster.restartOzoneManager(downedOM, true); OzoneManagerStateMachine omStateMachine = downedOM.getOmRatisServer() diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index c253e8b6e2c5..4fdd713a77de 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@ -146,6 +146,8 @@ private static void finalizeOMUpgrade() // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization cluster.getOzoneManager().getMetadataManager().getMetaTable() .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); OMUpgradeTestUtils.waitForFinalization(omClient); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index d7a5bbdef32a..7a97d2c28c2f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -328,6 +328,8 @@ private void finalizeOMUpgrade() throws Exception { // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization cluster.getOzoneManager().getMetadataManager().getMetaTable() .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); OMUpgradeTestUtils.waitForFinalization(omClient); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java index 6c4291b176fe..1f12ed10e95e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -95,6 +96,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omMetadataManager.getMetaTable().addCacheEntry( new CacheKey<>(APPARENT_VERSION_KEY), CacheValue.get(context.getIndex(), String.valueOf(apparentVersion))); + // Clear the finalization_in_progress key from the cache + omMetadataManager.getMetaTable().addCacheEntry( + new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY), CacheValue.get(System.nanoTime())); FinalizeUpgradeResponse omResponse = FinalizeUpgradeResponse.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java index 1d25f9096f4d..8dc981ba5eda 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -55,5 +56,7 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, APPARENT_VERSION_KEY, String.valueOf(serializedApparentVersion)); } + // Finalization has completed successfully, so the IN_PROGRESS Key should be removed from the database. + omMetadataManager.getMetaTable().deleteWithBatch(batchOperation, OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index 312740c660d7..b3d2799d03f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -27,8 +27,6 @@ import org.apache.hadoop.hdds.utils.BackgroundTask; import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; import org.apache.hadoop.hdds.utils.BackgroundTaskResult; -import org.apache.hadoop.hdds.utils.db.cache.CacheKey; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmClient; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; @@ -49,7 +47,6 @@ public class OMUpgradeFinalizeService extends BackgroundService { private static final TimeUnit INTERVAL_UNIT = TimeUnit.MILLISECONDS; private static final long TIMEOUT = 60000; private static final AtomicLong RUN_COUNT = new AtomicLong(0); - private static final CacheKey FINALIZATION_CACHE_KEY = new CacheKey<>(FINALIZATION_IN_PROGRESS_KEY); private final OzoneManager ozoneManager; private final OMVersionManager versionManager; @@ -109,9 +106,9 @@ public BackgroundTaskResult call() { try { // To finalize OM, first finalization needs to have been started. Then SCM needs to indicate that it has // completed its finalization work. Only once both of those things have happened can OM finalize. - CacheValue finalizationValue = - ozoneManager.getMetadataManager().getMetaTable().getCacheValue(FINALIZATION_CACHE_KEY); - if (finalizationValue.getCacheValue() == null) { + String finalizationInProgress = + ozoneManager.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY); + if (finalizationInProgress == null) { LOG.debug("OMUpgradeFinalizeService: skipping check — finalization is not in progress."); return BackgroundTaskResult.EmptyTaskResult.newResult(); } From 9afb2393bb3bd21270f443fbfe2c2e3c7c1c0bed Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 18 May 2026 17:49:35 +0100 Subject: [PATCH 16/37] Increase wait timeout for finalization --- .../java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java index 8f1fc41544a7..6eb852910f6f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java @@ -81,6 +81,6 @@ public static void waitForFinalization(OzoneManagerProtocol omClient) fail(e.getMessage()); } return false; - }, 2000, 20000); + }, 2000, 60000); } } From 4376bdffeb203924ebb00ed2084ab1beb24a23ad Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Mon, 18 May 2026 18:16:25 +0100 Subject: [PATCH 17/37] Debug messages --- .../apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 7 +++++++ .../hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index 3ec9a492accc..b740919da43f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -56,6 +56,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Upgrade testing for Bucket Layout Feature. @@ -75,6 +77,9 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class TestOMBucketLayoutUpgrade { + private static final Logger LOG = + LoggerFactory.getLogger(TestOMBucketLayoutUpgrade.class); + private static final int PRE_UPGRADE = 100; private static final int DURING_UPGRADE = 200; private static final int POST_UPGRADE = 300; @@ -161,10 +166,12 @@ void finalizeUpgrade() throws Exception { om.getMetadataManager().getMetaTable().addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); om.getMetadataManager().getMetaTable() .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + LOG.info("+++ Written the marker key to an OM"); } catch (RocksDatabaseException | CodecException e) { throw new RuntimeException(e); } }); + LOG.info("+++ Waiting for finalization to complete..."); waitForFinalization(omClient); final String expectedVersion = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index b3d2799d03f9..fee8313e273e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -109,9 +109,10 @@ public BackgroundTaskResult call() { String finalizationInProgress = ozoneManager.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY); if (finalizationInProgress == null) { - LOG.debug("OMUpgradeFinalizeService: skipping check — finalization is not in progress."); + LOG.info("+++ OMUpgradeFinalizeService: skipping check — finalization is not in progress."); return BackgroundTaskResult.EmptyTaskResult.newResult(); } + LOG.info("+++ OMUpgradeFinalizeService: finalization in progress and need finalization."); HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); if (upgradeStatus.getShouldFinalize()) { @@ -126,6 +127,8 @@ public BackgroundTaskResult call() { if (!response.getSuccess()) { LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis. {}", response.getMessage()); } + } else { + LOG.info("+++ SCM is not reporting as finalized"); } } catch (Exception e) { LOG.error("An exception occurred while trying to check the SCM Upgrade status or finalize OM", e); From 324cf4a107e9136323bc22dd849290c6c81190ec Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 19 May 2026 12:07:00 +0100 Subject: [PATCH 18/37] More debug --- .../apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 5de633082c2f..509cf180c58d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -1215,6 +1215,9 @@ public HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException { int healthyDatanodes = datanodeFinalizationCounts.getTotalHealthyDatanodes(); boolean shouldFinalize = scmFinalized && datanodeFinalizationCounts.allNodesFinalized(); + LOG.info("+++ SCMFinalized {} numDatanodesFinal {} total DNs {} shouldFinal {}", + scmFinalized, finalizedDatanodes, healthyDatanodes, shouldFinalize); + HddsProtos.UpgradeStatus result = HddsProtos.UpgradeStatus.newBuilder() .setScmFinalized(scmFinalized) .setNumDatanodesFinalized(finalizedDatanodes) From 93ac73147054d635f7a0e2d40830825745086804 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 19 May 2026 16:37:36 +0100 Subject: [PATCH 19/37] Fix issue where DN was defaulting to incorrect layout version --- .../apache/hadoop/ozone/container/common/DatanodeStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index 1b1be8df8b60..7f0f2436c7d3 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -95,7 +95,7 @@ public void setClusterId(String clusterId) throws IOException { * layout version is found on disk. */ private static int getDefaultLayoutVersion(ConfigurationSource conf) { - int defaultLayoutVersion = maxLayoutVersion(); + int defaultLayoutVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) { From 58b0a266c76bcfdbd2867d5d12f39079b424fe1a Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 19 May 2026 16:37:58 +0100 Subject: [PATCH 20/37] Revert "Increase wait timeout for finalization" This reverts commit 9afb2393bb3bd21270f443fbfe2c2e3c7c1c0bed. --- .../java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java index 6eb852910f6f..8f1fc41544a7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java @@ -81,6 +81,6 @@ public static void waitForFinalization(OzoneManagerProtocol omClient) fail(e.getMessage()); } return false; - }, 2000, 60000); + }, 2000, 20000); } } From 993fedc4846d54e0608390179de43048a43877a7 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 19 May 2026 16:38:14 +0100 Subject: [PATCH 21/37] Revert "Debug messages" This reverts commit 4376bdffeb203924ebb00ed2084ab1beb24a23ad. --- .../apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 7 ------- .../hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java | 5 +---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index b740919da43f..3ec9a492accc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -56,8 +56,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Upgrade testing for Bucket Layout Feature. @@ -77,9 +75,6 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class TestOMBucketLayoutUpgrade { - private static final Logger LOG = - LoggerFactory.getLogger(TestOMBucketLayoutUpgrade.class); - private static final int PRE_UPGRADE = 100; private static final int DURING_UPGRADE = 200; private static final int POST_UPGRADE = 300; @@ -166,12 +161,10 @@ void finalizeUpgrade() throws Exception { om.getMetadataManager().getMetaTable().addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); om.getMetadataManager().getMetaTable() .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); - LOG.info("+++ Written the marker key to an OM"); } catch (RocksDatabaseException | CodecException e) { throw new RuntimeException(e); } }); - LOG.info("+++ Waiting for finalization to complete..."); waitForFinalization(omClient); final String expectedVersion = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index fee8313e273e..b3d2799d03f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -109,10 +109,9 @@ public BackgroundTaskResult call() { String finalizationInProgress = ozoneManager.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY); if (finalizationInProgress == null) { - LOG.info("+++ OMUpgradeFinalizeService: skipping check — finalization is not in progress."); + LOG.debug("OMUpgradeFinalizeService: skipping check — finalization is not in progress."); return BackgroundTaskResult.EmptyTaskResult.newResult(); } - LOG.info("+++ OMUpgradeFinalizeService: finalization in progress and need finalization."); HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); if (upgradeStatus.getShouldFinalize()) { @@ -127,8 +126,6 @@ public BackgroundTaskResult call() { if (!response.getSuccess()) { LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis. {}", response.getMessage()); } - } else { - LOG.info("+++ SCM is not reporting as finalized"); } } catch (Exception e) { LOG.error("An exception occurred while trying to check the SCM Upgrade status or finalize OM", e); From 5f03e6bafd8b07eb14187e6049b582d7405377da Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 19 May 2026 16:38:26 +0100 Subject: [PATCH 22/37] Revert "More debug" This reverts commit 324cf4a107e9136323bc22dd849290c6c81190ec. --- .../apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 86a8a516c84b..806e0fed9a10 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -1215,9 +1215,6 @@ public HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException { int healthyDatanodes = datanodeFinalizationCounts.getTotalHealthyDatanodes(); boolean shouldFinalize = scmFinalized && datanodeFinalizationCounts.allNodesFinalized() && !scm.isInSafeMode(); - LOG.info("+++ SCMFinalized {} numDatanodesFinal {} total DNs {} shouldFinal {}", - scmFinalized, finalizedDatanodes, healthyDatanodes, shouldFinalize); - HddsProtos.UpgradeStatus result = HddsProtos.UpgradeStatus.newBuilder() .setScmFinalized(scmFinalized) .setNumDatanodesFinalized(finalizedDatanodes) From b6a07c7463c3241d8fbf6e0b21e3c02e2d08543d Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 20 May 2026 11:31:46 +0100 Subject: [PATCH 23/37] Fix failing finalize service tests --- .../om/upgrade/TestOMUpgradeFinalizeService.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 590bfbb00c5f..938177b1e58f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -33,8 +33,11 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.TypedTable; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -64,7 +67,7 @@ public class TestOMUpgradeFinalizeService { private OMUpgradeFinalizeService service; @BeforeEach - void setUp() { + void setUp() throws RocksDatabaseException, CodecException { ozoneManager = mock(OzoneManager.class); OMMetadataManager metadataManager = mock(OMMetadataManager.class); metaTable = mock(TypedTable.class); @@ -73,7 +76,7 @@ void setUp() { when(ozoneManager.getMetadataManager()).thenReturn(metadataManager); when(metadataManager.getMetaTable()).thenReturn(metaTable); // For most tests, set the finalization command as having been received - when(metaTable.getCacheValue(any())).thenReturn(CacheValue.get(1, "ignored")); + when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn("ignored"); versionManager = mock(OMVersionManager.class); // preExecute() calls ozoneManager.getVersionManager().getApparentVersion().serialize() @@ -138,14 +141,14 @@ void testFinalizationTriggeredWhenScmIsFinalizedAndFinalizationInProgress() thro .build(); when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); // Finalization command not given yet - when(metaTable.getCacheValue(any())).thenReturn(null); + when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn(null); service.runPeriodicalTaskNow(); verifyNoInteractions(containerClient); verifyNoInteractions(omRatisServer); - when(metaTable.getCacheValue(any())).thenReturn(CacheValue.get(1, "ignored")); + when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn("ignored"); service.runPeriodicalTaskNow(); verify(containerClient).queryUpgradeStatus(); From b02c11616f8125436bbde743f63e6554b8200fa9 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 20 May 2026 11:56:43 +0100 Subject: [PATCH 24/37] Fix checkstyle --- .../hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index 938177b1e58f..a3aa7be8b5f0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hdds.utils.db.CodecException; import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.TypedTable; -import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.om.OMMetadataManager; From 9352c04cc7bb7eb5bc12a1980bc3b786c4517b3e Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 21 May 2026 14:50:39 +0100 Subject: [PATCH 25/37] Fix failing container upgrade test with Ethan's patch --- .../schemaupgrade/UpgradeContainerSchema.java | 35 ++++++++++--------- .../datanode/schemaupgrade/UpgradeUtils.java | 26 -------------- 2 files changed, 18 insertions(+), 43 deletions(-) diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java index 7add67421fae..ffd9dfb8fa45 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -50,6 +50,7 @@ import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.container.common.DatanodeStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; @@ -63,6 +64,7 @@ import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeStore; import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl; +import org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager; import org.apache.hadoop.ozone.repair.RepairTool; import org.apache.hadoop.util.Time; import picocli.CommandLine; @@ -131,22 +133,21 @@ public void execute() throws Exception { DatanodeDetails dnDetail = UpgradeUtils.getDatanodeDetails(configuration); - Pair layoutFeature = - UpgradeUtils.getLayoutFeature(dnDetail, configuration); - final HDDSLayoutFeature softwareLayoutFeature = layoutFeature.getLeft(); - final HDDSLayoutFeature metadataLayoutFeature = layoutFeature.getRight(); - final int needLayoutVersion = - HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(); - - if (metadataLayoutFeature.layoutVersion() < needLayoutVersion || - softwareLayoutFeature.layoutVersion() < needLayoutVersion) { - fatal( - "Please upgrade your software version, no less than %s," + - " current metadata layout version is %s," + - " software layout version is %s", - HDDSLayoutFeature.DATANODE_SCHEMA_V3.toString(), - metadataLayoutFeature.toString(), softwareLayoutFeature.toString()); - return; + DatanodeStorage storage = new DatanodeStorage(configuration, dnDetail.getUuidString()); + try (DatanodeVersionManager versionManager = new DatanodeVersionManager(storage, null)) { + // Ensure repair tool is not run in a newer version that supports schema V3 while the datanode that will read the + // containers does not. + if (!HDDSLayoutFeature.DATANODE_SCHEMA_V3.isSupportedBy(HDDSVersion.SOFTWARE_VERSION)) { + fatal("Please upgrade your software version to at least %s, current software version is %s", + HDDSLayoutFeature.DATANODE_SCHEMA_V3, HDDSVersion.SOFTWARE_VERSION); + return; + } + + + if (!versionManager.isAllowed(HDDSLayoutFeature.DATANODE_SCHEMA_V3)) { + fatal("Please finalize the cluster to enable support for Datanode container schema V3"); + return; + } } if (!Strings.isNullOrEmpty(volume)) { diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java index 164f91b6a327..5c559c52f285 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java @@ -29,13 +29,9 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.HddsServerUtil; -import org.apache.hadoop.ozone.container.common.DatanodeStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; @@ -93,28 +89,6 @@ public static boolean createFile(File file) throws IOException { return file.exists(); } - public static Pair getLayoutFeature( - DatanodeDetails dnDetail, OzoneConfiguration conf) throws IOException { - DatanodeStorage layoutStorage = - new DatanodeStorage(conf, dnDetail.getUuidString()); - HDDSLayoutVersionManager layoutVersionManager = - new HDDSLayoutVersionManager(layoutStorage.getApparentVersion(), null, null); - - final int metadataLayoutVersion = - layoutVersionManager.getMetadataLayoutVersion(); - final HDDSLayoutFeature metadataLayoutFeature = - (HDDSLayoutFeature) layoutVersionManager.getFeature( - metadataLayoutVersion); - - final int softwareLayoutVersion = - layoutVersionManager.getSoftwareLayoutVersion(); - final HDDSLayoutFeature softwareLayoutFeature = - (HDDSLayoutFeature) layoutVersionManager.getFeature( - softwareLayoutVersion); - - return Pair.of(softwareLayoutFeature, metadataLayoutFeature); - } - public static List getAllVolume(DatanodeDetails detail, OzoneConfiguration configuration) throws IOException { final MutableVolumeSet dataVolumeSet = getHddsVolumes(configuration, StorageVolume.VolumeType.DATA_VOLUME, From 7282b36044044ad64cc185f001fcbde2ad7457d0 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 21 May 2026 14:51:45 +0100 Subject: [PATCH 26/37] Revert "Fix issue where DN was defaulting to incorrect layout version" This reverts commit 93ac73147054d635f7a0e2d40830825745086804. --- .../apache/hadoop/ozone/container/common/DatanodeStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index 7f0f2436c7d3..1b1be8df8b60 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; -import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -95,7 +95,7 @@ public void setClusterId(String clusterId) throws IOException { * layout version is found on disk. */ private static int getDefaultLayoutVersion(ConfigurationSource conf) { - int defaultLayoutVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); + int defaultLayoutVersion = maxLayoutVersion(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) { From 0e6a85f0f59529c427fdcabac1c501d941c48b6a Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 21 May 2026 14:52:53 +0100 Subject: [PATCH 27/37] Fix DN default versions --- .../ozone/container/common/DatanodeStorage.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index 1b1be8df8b60..fe2677834378 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -43,7 +43,7 @@ public class DatanodeStorage extends Storage { public DatanodeStorage(ConfigurationSource conf, String dataNodeId) throws IOException { super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf), - DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultLayoutVersion(conf)); + DATANODE_LAYOUT_VERSION_DIR, dataNodeId, getDefaultApparentVersion(conf)); } public DatanodeStorage(OzoneConfiguration conf, String dataNodeId, @@ -56,7 +56,7 @@ public DatanodeStorage(OzoneConfiguration conf, String dataNodeId, public DatanodeStorage(ConfigurationSource conf) throws IOException { super(NodeType.DATANODE, ServerUtils.getOzoneMetaDirPath(conf), - DATANODE_LAYOUT_VERSION_DIR, getDefaultLayoutVersion(conf)); + DATANODE_LAYOUT_VERSION_DIR, getDefaultApparentVersion(conf)); } @Override @@ -94,15 +94,14 @@ public void setClusterId(String clusterId) throws IOException { * @return The layout version that should be used for the datanode if no * layout version is found on disk. */ - private static int getDefaultLayoutVersion(ConfigurationSource conf) { - int defaultLayoutVersion = maxLayoutVersion(); + private static int getDefaultApparentVersion(ConfigurationSource conf) { + int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) { - defaultLayoutVersion = - HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(); + defaultApparentVersion = HDDSLayoutFeature.INITIAL_VERSION.layoutVersion(); } - return defaultLayoutVersion; + return defaultApparentVersion; } } From 287a6963fa2c434f7e47cda3b292b8140fe7c7eb Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 28 May 2026 11:39:16 +0100 Subject: [PATCH 28/37] Revert unintentional log format change --- .../integration-test/src/test/resources/auditlog.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties index bc5e6cd2bd00..fb9fb63feec1 100644 --- a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties +++ b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties @@ -62,7 +62,7 @@ appender.audit.type = File appender.audit.name = AUDITLOG appender.audit.fileName=audit.log appender.audit.layout.type=PatternLayout -appender.audit.layout.pattern= %-5level | %c{1} | %C | %msg%n +appender.audit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n loggers=audit, systemaudit logger.audit.name=OMAudit @@ -74,7 +74,7 @@ appender.systemaudit.type = File appender.systemaudit.name = SYSTEMAUDITLOG appender.systemaudit.fileName=system_audit.log appender.systemaudit.layout.type=PatternLayout -appender.systemaudit.layout.pattern= %-5level | %c{1} | %C | %msg%n +appender.systemaudit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n logger.systemaudit.name=OMSystemAudit logger.systemaudit.level = INFO From 1dba392e907b6202270272c475f6ac7419d0af2a Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 28 May 2026 11:44:45 +0100 Subject: [PATCH 29/37] Also delete system audit log on test finish --- .../java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java | 4 ++++ .../org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java | 1 + 2 files changed, 5 insertions(+) diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java index 81a406d01cfc..3c438f07c8a5 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java @@ -97,4 +97,8 @@ public static void truncateAuditLogFile() throws IOException { public static void deleteAuditLogFile() { FileUtils.deleteQuietly(new File(AUDITLOG_FILENAME)); } + + public static void deleteSystemAuditLogFile() { + FileUtils.deleteQuietly(new File(SYSTEM_AUDITLOG_FILENAME)); + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index e6c84c390d3b..03f1786415cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -66,6 +66,7 @@ public void setup() throws Exception { @AfterAll public static void shutdown() { AuditLogTestUtils.deleteAuditLogFile(); + AuditLogTestUtils.deleteSystemAuditLogFile(); } @Test From 66fd9f5716f229abc69ab17c676219ba89dc3403 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 28 May 2026 11:48:45 +0100 Subject: [PATCH 30/37] Use ratis transaction as epoch --- .../ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java index 1f12ed10e95e..0aab6e4cdc39 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java @@ -98,7 +98,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut CacheValue.get(context.getIndex(), String.valueOf(apparentVersion))); // Clear the finalization_in_progress key from the cache omMetadataManager.getMetaTable().addCacheEntry( - new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY), CacheValue.get(System.nanoTime())); + new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY), CacheValue.get(context.getIndex())); FinalizeUpgradeResponse omResponse = FinalizeUpgradeResponse.newBuilder() From fd950a7632dbbd371833595f4dbd5a7e027b6ef4 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 28 May 2026 11:52:55 +0100 Subject: [PATCH 31/37] Use same random clientID --- .../hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index b3d2799d03f9..f0d00ac3cbbc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -52,6 +52,7 @@ public class OMUpgradeFinalizeService extends BackgroundService { private final OMVersionManager versionManager; private final ScmClient scmClient; private final AtomicBoolean stopInitiated = new AtomicBoolean(false); + private final ClientId clientId = ClientId.randomId(); /** * Creates an {@code OMUpgradeFinalizeService} with a custom check interval. @@ -119,10 +120,10 @@ public BackgroundTaskResult call() { OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) - .setClientId(ozoneManager.getOMNodeId()) + .setClientId(clientId.toString()) .build(); OzoneManagerProtocolProtos.OMResponse response = OzoneManagerRatisUtils.submitRequest( - ozoneManager, omRequest, ClientId.randomId(), RUN_COUNT.getAndIncrement()); + ozoneManager, omRequest, clientId, RUN_COUNT.getAndIncrement()); if (!response.getSuccess()) { LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis. {}", response.getMessage()); } From f9f0be1f1e78632b377aad25a767efdb2acb9bd1 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 28 May 2026 12:10:23 +0100 Subject: [PATCH 32/37] Always increment run count and add to logs --- .../om/upgrade/OMUpgradeFinalizeService.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index f0d00ac3cbbc..93926ba65944 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -99,8 +99,9 @@ private class UpgradeStatusCheckTask implements BackgroundTask { @Override public BackgroundTaskResult call() { + final long run = RUN_COUNT.incrementAndGet(); if (!ozoneManager.isLeaderReady()) { - LOG.debug("OMUpgradeFinalizeService: skipping check — not the leader."); + LOG.debug("OMUpgradeFinalizeService: skipping check — not the leader. Run count {}", run); return BackgroundTaskResult.EmptyTaskResult.newResult(); } if (versionManager.needsFinalization()) { @@ -110,27 +111,33 @@ public BackgroundTaskResult call() { String finalizationInProgress = ozoneManager.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY); if (finalizationInProgress == null) { - LOG.debug("OMUpgradeFinalizeService: skipping check — finalization is not in progress."); + LOG.debug("OMUpgradeFinalizeService: skipping check — finalization is not in progress. Run count {}", run); return BackgroundTaskResult.EmptyTaskResult.newResult(); } HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); if (upgradeStatus.getShouldFinalize()) { - LOG.info("The SCM Upgrade has been finalized. OM will now finalize"); + LOG.info("The SCM Upgrade has been finalized. OM will now finalize. Run count {}", run); OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) .setClientId(clientId.toString()) .build(); OzoneManagerProtocolProtos.OMResponse response = OzoneManagerRatisUtils.submitRequest( - ozoneManager, omRequest, clientId, RUN_COUNT.getAndIncrement()); + ozoneManager, omRequest, clientId, run); if (!response.getSuccess()) { - LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis. {}", response.getMessage()); + LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis. {}. Run count {}", + response.getMessage(), run); } + } else { + LOG.debug("The SCM Upgrade has not been finalized. Run count {}", run); } } catch (Exception e) { - LOG.error("An exception occurred while trying to check the SCM Upgrade status or finalize OM", e); + LOG.error("An exception occurred while trying to check the SCM Upgrade status or finalize OM. Run count {}", + run, e); } + } else { + LOG.debug("Finalization is not in progress. Run count {}", run); } return BackgroundTaskResult.EmptyTaskResult.newResult(); } From c8325a792426a135b53b7e364db419c703f053e2 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Thu, 28 May 2026 16:08:03 +0100 Subject: [PATCH 33/37] Fix log4j settings --- .../src/test/resources/auditlog.properties | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties index fb9fb63feec1..73be606e9061 100644 --- a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties +++ b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties @@ -64,11 +64,13 @@ appender.audit.fileName=audit.log appender.audit.layout.type=PatternLayout appender.audit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -loggers=audit, systemaudit +loggers=audit,systemaudit + logger.audit.name=OMAudit -logger.audit.level = INFO -logger.audit.appenderRefs = audit -logger.audit.appenderRef.file.ref = AUDITLOG +logger.audit.level=INFO +logger.audit.additivity=false +logger.audit.appenderRefs=audit +logger.audit.appenderRef.audit.ref=AUDITLOG appender.systemaudit.type = File appender.systemaudit.name = SYSTEMAUDITLOG @@ -77,9 +79,10 @@ appender.systemaudit.layout.type=PatternLayout appender.systemaudit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n logger.systemaudit.name=OMSystemAudit -logger.systemaudit.level = INFO -logger.systemaudit.appenderRefs = audit -logger.systemaudit.appenderRef.file.ref = SYSTEMAUDITLOG +logger.systemaudit.level=INFO +logger.systemaudit.additivity=false +logger.systemaudit.appenderRefs=systemaudit +logger.systemaudit.appenderRef.systemaudit.ref=SYSTEMAUDITLOG rootLogger.level = INFO rootLogger.appenderRefs = stdout From 2fa62f2e0914786a6b4ab009117f7848fa959675 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Fri, 29 May 2026 17:48:07 +0100 Subject: [PATCH 34/37] Fix bad merge affecting DN version --- .../apache/hadoop/ozone/container/common/DatanodeStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index 53b62afc3e16..fe2677834378 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -95,7 +95,7 @@ public void setClusterId(String clusterId) throws IOException { * layout version is found on disk. */ private static int getDefaultApparentVersion(ConfigurationSource conf) { - int defaultApparentVersion = maxLayoutVersion(); + int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) { From 9d3916854362d03b144dd335fa36ce61255dcc27 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Tue, 2 Jun 2026 14:09:33 +0100 Subject: [PATCH 35/37] Fixes to auditlog.properties --- .../src/test/resources/auditlog.properties | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties index 73be606e9061..d7051b6cfdae 100644 --- a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties +++ b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties @@ -56,7 +56,7 @@ appenders = console, audit, systemaudit appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %-5level | %c{1} | %msg%n +appender.console.layout.pattern = %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n appender.audit.type = File appender.audit.name = AUDITLOG @@ -69,8 +69,9 @@ loggers=audit,systemaudit logger.audit.name=OMAudit logger.audit.level=INFO logger.audit.additivity=false -logger.audit.appenderRefs=audit +logger.audit.appenderRefs=audit,console logger.audit.appenderRef.audit.ref=AUDITLOG +logger.audit.appenderRef.console.ref=STDOUT appender.systemaudit.type = File appender.systemaudit.name = SYSTEMAUDITLOG @@ -81,8 +82,9 @@ appender.systemaudit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %th logger.systemaudit.name=OMSystemAudit logger.systemaudit.level=INFO logger.systemaudit.additivity=false -logger.systemaudit.appenderRefs=systemaudit +logger.systemaudit.appenderRefs=systemaudit,console logger.systemaudit.appenderRef.systemaudit.ref=SYSTEMAUDITLOG +logger.systemaudit.appenderRef.console.ref=STDOUT rootLogger.level = INFO rootLogger.appenderRefs = stdout From 1f3e6b22b19d830526b6fdd2d0d8f5e6101b4327 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 3 Jun 2026 15:27:27 +0100 Subject: [PATCH 36/37] Add tests for finalize key manipulation in request / response handling --- .../upgrade/TestOMFinalizeUpgradeRequest.java | 85 +++++++++++++++++++ .../TestOMFinalizeUpgradeResponse.java | 80 +++++++++++++++++ .../om/response/upgrade/package-info.java | 21 +++++ 3 files changed, 186 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java new file mode 100644 index 000000000000..8a8a97de84ac --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.upgrade; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Collections; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; +import org.apache.hadoop.ozone.om.request.key.TestOMKeyRequest; +import org.apache.hadoop.ozone.om.upgrade.OMVersionManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.server.protocol.TermIndex; +import org.junit.jupiter.api.Test; + +/** + * Tests for the OMFinalizeUpgradeRequest class. + */ +public class TestOMFinalizeUpgradeRequest extends TestOMKeyRequest { + + @Test + public void testFinalizationInProgressKeyRemoved() throws IOException { + OMVersionManager omVersionManager = mock(OMVersionManager.class); + when(omVersionManager.getApparentVersion()).thenReturn(OzoneManagerVersion.DEFAULT_VERSION); + when(ozoneManager.getVersionManager()).thenReturn(omVersionManager); + when(ozoneManager.finalizeUpgrade(any())).thenReturn(new UpgradeFinalization.StatusAndMessages( + UpgradeFinalization.Status.FINALIZATION_IN_PROGRESS, Collections.singletonList("Finalization in progress"))); + + omMetadataManager.getMetaTable().put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignored"); + omMetadataManager.getMetaTable().addCacheEntry( + new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY), CacheValue.get(1, "ignored")); + + String progressKey = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + + assertNotNull(progressKey); + submitRequest(); + + progressKey = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + assertNull(progressKey); + } + + private void submitRequest() throws IOException { + OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) + .setClientId(ClientId.randomId().toString()) + .build(); + + OMFinalizeUpgradeRequest request = new OMFinalizeUpgradeRequest(omRequest); + ExecutionContext context = ExecutionContext.of(1, TermIndex.INITIAL_VALUE); + + OMFinalizeUpgradeRequest finalizeUpgradeRequest = new OMFinalizeUpgradeRequest(omRequest); + OzoneManagerProtocolProtos.OMRequest modifiedOmRequest = request.preExecute(ozoneManager); + + // Will not be equal, as UserInfo will be set. + assertNotEquals(omRequest, modifiedOmRequest); + request.validateAndUpdateCache(ozoneManager, context); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java new file mode 100644 index 000000000000..15ad3c848b5f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.upgrade; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.IOException; +import java.nio.file.Path; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for the OMFinalizeUpgradeResponse class. + */ +public class TestOMFinalizeUpgradeResponse { + + @TempDir + private Path folder; + private OMMetadataManager omMetadataManager; + private BatchOperation batchOperation; + + @BeforeEach + public void setup() throws Exception { + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, + folder.toAbsolutePath().toString()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + batchOperation = omMetadataManager.getStore().initBatchOperation(); + } + + @Test + public void testFinalizationInProgressKeyRemoved() throws IOException { + // Add the in progress key which would normally have been added by the start finalization command + omMetadataManager.getMetaTable().put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignored"); + + String value = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + assertEquals("ignored", value); + + OMFinalizeUpgradeResponse finalizeUpgradeResponse = new OMFinalizeUpgradeResponse(createRequest(), 1); + finalizeUpgradeResponse.addToDBBatch(omMetadataManager, batchOperation); + + omMetadataManager.getStore().commitBatchOperation(batchOperation); + + // Ensure the key is removed as expected + value = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + assertNull(value); + } + + private OzoneManagerProtocolProtos.OMResponse createRequest() { + return OzoneManagerProtocolProtos.OMResponse.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .build(); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java new file mode 100644 index 000000000000..ef001e338eb9 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package contains test classes for upgrade responses. + */ +package org.apache.hadoop.ozone.om.response.upgrade; From 4eaa1a8ea4680cca2283571e0d4540036a4e2949 Mon Sep 17 00:00:00 2001 From: S O'Donnell Date: Wed, 3 Jun 2026 15:53:25 +0100 Subject: [PATCH 37/37] Fix find bugs --- .../ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java index 8a8a97de84ac..863ead019ed5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java @@ -74,7 +74,6 @@ private void submitRequest() throws IOException { OMFinalizeUpgradeRequest request = new OMFinalizeUpgradeRequest(omRequest); ExecutionContext context = ExecutionContext.of(1, TermIndex.INITIAL_VALUE); - OMFinalizeUpgradeRequest finalizeUpgradeRequest = new OMFinalizeUpgradeRequest(omRequest); OzoneManagerProtocolProtos.OMRequest modifiedOmRequest = request.preExecute(ozoneManager); // Will not be equal, as UserInfo will be set.