diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 5d0b826c17e1..1aa5961eb368 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -439,6 +439,9 @@ public final class OzoneConsts { // Apparent Version written into Meta Table ONLY during finalization. // The name "layout version" is kept for backwards compatibility. public static final String APPARENT_VERSION_KEY = "#LAYOUTVERSION"; + // Key written into the Meta table when finalization is needed and a finalization command has been received + // to trigger the process + public static final String FINALIZATION_IN_PROGRESS_KEY = "#FINALIZATION_IN_PROGRESS"; // Kerberos constants public static final String KERBEROS_CONFIG_VALUE = "kerberos"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index aeda046dcd74..dc16b29b77a5 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -5028,4 +5028,10 @@ OZONE, RATIS, OM The maximum number of events that can be pending in OM Ratis. + + ozone.om.upgrade.finalization.check.interval + 1m + OM + If OM is unfinalized, how frequently it should poll SCM to trigger finalization + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java index 53b62afc3e16..fe2677834378 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/DatanodeStorage.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.container.common; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; import static org.apache.hadoop.ozone.OzoneConsts.DATANODE_LAYOUT_VERSION_DIR; import java.io.File; import java.io.IOException; import java.util.Properties; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; @@ -95,7 +95,7 @@ public void setClusterId(String clusterId) throws IOException { * layout version is found on disk. */ private static int getDefaultApparentVersion(ConfigurationSource conf) { - int defaultApparentVersion = maxLayoutVersion(); + int defaultApparentVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); File dnIdFile = new File(HddsServerUtil.getDatanodeIdFilePath(conf)); if (dnIdFile.exists()) { diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java index 0b05d4d01f0e..3c438f07c8a5 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java @@ -32,6 +32,7 @@ */ public final class AuditLogTestUtils { private static final String AUDITLOG_FILENAME = "audit.log"; + private static final String SYSTEM_AUDITLOG_FILENAME = "system_audit.log"; private AuditLogTestUtils() { } @@ -55,8 +56,26 @@ public static void verifyAuditLog(AuditAction action, 1000, 10000); } + /** + * Searches for the given action in the system audit log file. + */ + public static void verifySystemAuditLog(AuditAction action, + AuditEventStatus eventStatus) throws InterruptedException, TimeoutException { + waitFor( + () -> fileContains(SYSTEM_AUDITLOG_FILENAME, action.getAction(), eventStatus.getStatus()), + 1000, 10000); + } + public static boolean auditLogContains(String... strings) { - File file = new File(AUDITLOG_FILENAME); + return fileContains(AUDITLOG_FILENAME, strings); + } + + public static boolean systemAuditLogContains(String... strings) { + return fileContains(SYSTEM_AUDITLOG_FILENAME, strings); + } + + private static boolean fileContains(String filename, String... strings) { + File file = new File(filename); try { String contents = FileUtils.readFileToString(file, UTF_8); for (String s : strings) { @@ -72,9 +91,14 @@ public static boolean auditLogContains(String... strings) { public static void truncateAuditLogFile() throws IOException { Files.write(Paths.get(AUDITLOG_FILENAME), new byte[0]); + Files.write(Paths.get(SYSTEM_AUDITLOG_FILENAME), new byte[0]); } public static void deleteAuditLogFile() { FileUtils.deleteQuietly(new File(AUDITLOG_FILENAME)); } + + public static void deleteSystemAuditLogFile() { + FileUtils.deleteQuietly(new File(SYSTEM_AUDITLOG_FILENAME)); + } } diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java index 7add67421fae..ffd9dfb8fa45 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeContainerSchema.java @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -50,6 +50,7 @@ import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.container.common.DatanodeStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; @@ -63,6 +64,7 @@ import org.apache.hadoop.ozone.container.metadata.DatanodeSchemaThreeDBDefinition; import org.apache.hadoop.ozone.container.metadata.DatanodeStore; import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl; +import org.apache.hadoop.ozone.container.upgrade.DatanodeVersionManager; import org.apache.hadoop.ozone.repair.RepairTool; import org.apache.hadoop.util.Time; import picocli.CommandLine; @@ -131,22 +133,21 @@ public void execute() throws Exception { DatanodeDetails dnDetail = UpgradeUtils.getDatanodeDetails(configuration); - Pair layoutFeature = - UpgradeUtils.getLayoutFeature(dnDetail, configuration); - final HDDSLayoutFeature softwareLayoutFeature = layoutFeature.getLeft(); - final HDDSLayoutFeature metadataLayoutFeature = layoutFeature.getRight(); - final int needLayoutVersion = - HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(); - - if (metadataLayoutFeature.layoutVersion() < needLayoutVersion || - softwareLayoutFeature.layoutVersion() < needLayoutVersion) { - fatal( - "Please upgrade your software version, no less than %s," + - " current metadata layout version is %s," + - " software layout version is %s", - HDDSLayoutFeature.DATANODE_SCHEMA_V3.toString(), - metadataLayoutFeature.toString(), softwareLayoutFeature.toString()); - return; + DatanodeStorage storage = new DatanodeStorage(configuration, dnDetail.getUuidString()); + try (DatanodeVersionManager versionManager = new DatanodeVersionManager(storage, null)) { + // Ensure repair tool is not run in a newer version that supports schema V3 while the datanode that will read the + // containers does not. + if (!HDDSLayoutFeature.DATANODE_SCHEMA_V3.isSupportedBy(HDDSVersion.SOFTWARE_VERSION)) { + fatal("Please upgrade your software version to at least %s, current software version is %s", + HDDSLayoutFeature.DATANODE_SCHEMA_V3, HDDSVersion.SOFTWARE_VERSION); + return; + } + + + if (!versionManager.isAllowed(HDDSLayoutFeature.DATANODE_SCHEMA_V3)) { + fatal("Please finalize the cluster to enable support for Datanode container schema V3"); + return; + } } if (!Strings.isNullOrEmpty(volume)) { diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java index 164f91b6a327..5c559c52f285 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java @@ -29,13 +29,9 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.hdds.utils.HddsServerUtil; -import org.apache.hadoop.ozone.container.common.DatanodeStorage; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; @@ -93,28 +89,6 @@ public static boolean createFile(File file) throws IOException { return file.exists(); } - public static Pair getLayoutFeature( - DatanodeDetails dnDetail, OzoneConfiguration conf) throws IOException { - DatanodeStorage layoutStorage = - new DatanodeStorage(conf, dnDetail.getUuidString()); - HDDSLayoutVersionManager layoutVersionManager = - new HDDSLayoutVersionManager(layoutStorage.getApparentVersion(), null, null); - - final int metadataLayoutVersion = - layoutVersionManager.getMetadataLayoutVersion(); - final HDDSLayoutFeature metadataLayoutFeature = - (HDDSLayoutFeature) layoutVersionManager.getFeature( - metadataLayoutVersion); - - final int softwareLayoutVersion = - layoutVersionManager.getSoftwareLayoutVersion(); - final HDDSLayoutFeature softwareLayoutFeature = - (HDDSLayoutFeature) layoutVersionManager.getFeature( - softwareLayoutVersion); - - return Pair.of(softwareLayoutFeature, metadataLayoutFeature); - } - public static List getAllVolume(DatanodeDetails detail, OzoneConfiguration configuration) throws IOException { final MutableVolumeSet dataVolumeSet = getHddsVolumes(configuration, StorageVolume.VolumeType.DATA_VOLUME, diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index b9ccfedb40d0..a9ec011ac07a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -698,6 +698,10 @@ public final class OMConfigKeys { "ozone.om.ratis.events.max.limit"; public static final int OZONE_OM_RATIS_EVENTS_MAX_LIMIT_DEFAULT = 100; + public static final String OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL = + "ozone.om.upgrade.finalization.check.interval"; + public static final String OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL_DEFAULT = "1m"; + /** * Never constructed. */ diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java index 22d8f60920fa..ff5680a9fd39 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java @@ -29,16 +29,11 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; -import static org.apache.ozone.test.LambdaTestUtils.await; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; -import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataOutputStream; @@ -53,19 +48,21 @@ import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl; import org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OMUpgradeTestUtils; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -93,9 +90,6 @@ public class TestHSyncUpgrade { private static final int SERVICE_INTERVAL = 100; private static final int EXPIRE_THRESHOLD_MS = 140; - private static final int POLL_INTERVAL_MILLIS = 500; - private static final int POLL_MAX_WAIT_MILLIS = 120_000; - @BeforeEach public void init() throws Exception { final BucketLayout layout = BUCKET_LAYOUT; @@ -118,6 +112,7 @@ public void init() throws Exception { EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS); conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s"); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) @@ -217,20 +212,12 @@ private void finalizeOMUpgrade() throws Exception { // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); - UpgradeFinalization.StatusAndMessages finalizationResponse = - omClient.finalizeUpgrade(upgradeClientID); - - // The status should transition as soon as the client call above returns - assertTrue(isStarting(finalizationResponse.status())); - // Wait for the finalization to be marked as done. - // 10s timeout should be plenty. - await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> { - final UpgradeFinalization.StatusAndMessages progress = - omClient.queryUpgradeFinalizationProgress( - upgradeClientID, false, false); - return isDone(progress.status()); - }); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + OMUpgradeTestUtils.waitForFinalization(omClient); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java index 3f4120a48420..3ec9a492accc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java @@ -33,8 +33,11 @@ import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -42,7 +45,6 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.ozone.test.LambdaTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -89,6 +91,7 @@ class TestOMBucketLayoutUpgrade { void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, fromVersion.serialize()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); String omServiceId = UUID.randomUUID().toString(); MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf); builder.setOMServiceId(omServiceId) @@ -152,10 +155,16 @@ void allowsLegacyBucketBeforeUpgrade() throws Exception { @Test @Order(DURING_UPGRADE) void finalizeUpgrade() throws Exception { - UpgradeFinalization.StatusAndMessages response = - omClient.finalizeUpgrade("finalize-test"); - System.out.println("Finalization Messages : " + response.msgs()); - + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManagersList().forEach(om -> { + try { + om.getMetadataManager().getMetaTable().addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + om.getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + } catch (RocksDatabaseException | CodecException e) { + throw new RuntimeException(e); + } + }); waitForFinalization(omClient); final String expectedVersion = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index de2bc98f10c9..cdb40ed36d9e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -297,7 +297,7 @@ public void testInstallSnapshot(@TempDir Path tempDir) throws Exception { String toMatch = String.format( "op=DB_CHECKPOINT_INSTALL {\"leaderId\":\"%s\",\"term\":\"%d\",\"lastAppliedIndex\":\"%d\"}", leaderOMNodeId, leaderOMSnapshotTermIndex, followerOMLastAppliedIndex); - assertTrue(AuditLogTestUtils.auditLogContains(toMatch)); + assertTrue(AuditLogTestUtils.systemAuditLogContains(toMatch)); // Read & Write after snapshot installed. List newKeys = writeKeys(1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index 476ae16af055..03f1786415cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -25,14 +25,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import java.io.IOException; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogTestUtils; @@ -40,10 +44,8 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization.StatusAndMessages; import org.apache.ratis.util.LifeCycle; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -64,11 +66,14 @@ public void setup() throws Exception { @AfterAll public static void shutdown() { AuditLogTestUtils.deleteAuditLogFile(); + AuditLogTestUtils.deleteSystemAuditLogFile(); } @Test void testOMUpgradeFinalizationWithOneOMDown() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, INITIAL_VERSION.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); try (MiniOzoneHAClusterImpl cluster = newCluster(conf)) { cluster.waitForClusterToBeReady(); @@ -77,7 +82,7 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { for (OzoneManager om : runningOms) { assertEquals(INITIAL_VERSION, om.getVersionManager().getApparentVersion()); // The OMs have not been finalized yet, so no version has been written to the DB. - Assertions.assertNull(om.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY)); + assertNull(om.getMetadataManager().getMetaTable().get(APPARENT_VERSION_KEY)); } final int shutdownOMIndex = 2; @@ -93,15 +98,25 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception { long prepareIndex = omClient.prepareOzoneManager(120L, 5L); assertClusterPrepared(prepareIndex, runningOms); AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_PREPARE, AuditEventStatus.SUCCESS); - omClient.cancelOzoneManagerPrepare(); AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_CANCEL, AuditEventStatus.SUCCESS); - StatusAndMessages response = - omClient.finalizeUpgrade("finalize-test"); - System.out.println("Finalization Messages : " + response.msgs()); - AuditLogTestUtils.verifyAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManagersList().forEach(om -> { + try { + om.getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + om.getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + } catch (RocksDatabaseException | CodecException e) { + throw new RuntimeException(e); + } + }); waitForFinalization(omClient); + AuditLogTestUtils.verifySystemAuditLog(OMAction.UPGRADE_FINALIZE, AuditEventStatus.SUCCESS); + // Ensure the finalization in progress key has been removed. + assertNull(cluster.getOzoneManager().getMetadataManager() + .getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)); cluster.restartOzoneManager(downedOM, true); OzoneManagerStateMachine omStateMachine = downedOM.getOmRatisServer() diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index f1102777327c..4fdd713a77de 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@ -18,13 +18,9 @@ package org.apache.hadoop.ozone.om.multitenant; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MULTITENANCY_ENABLED; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import com.google.protobuf.ServiceException; import java.io.IOException; @@ -41,15 +37,15 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.rpc.RpcClient; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl; import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OMUpgradeTestUtils; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.protocol.S3Auth; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; -import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.LambdaTestUtils.VoidCallable; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -77,6 +73,7 @@ public static void initClusterProvider() throws Exception { OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true); conf.setBoolean(OZONE_OM_MULTITENANCY_ENABLED, true); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.INITIAL_VERSION.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf) .withoutDatanodes(); cluster = builder.build(); @@ -146,27 +143,12 @@ private static void finalizeOMUpgrade() // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); - UpgradeFinalization.StatusAndMessages finalizationResponse = - omClient.finalizeUpgrade(upgradeClientID); - - // The status should transition as soon as the client call above returns - assertTrue(isStarting(finalizationResponse.status())); - - // Wait for the finalization to be marked as done. - // 10s timeout should be plenty. - GenericTestUtils.waitFor(() -> { - try { - final UpgradeFinalization.StatusAndMessages progress = - omClient.queryUpgradeFinalizationProgress( - upgradeClientID, false, false); - return isDone(progress.status()); - } catch (IOException e) { - fail("Unexpected exception while waiting for " - + "the OM upgrade to finalize: " + e.getMessage()); - } - return false; - }, 500, 10000); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + OMUpgradeTestUtils.waitForFinalization(omClient); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index becf7fb9e5ac..7a97d2c28c2f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -50,8 +50,6 @@ import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG; import static org.apache.ozone.test.LambdaTestUtils.await; import static org.assertj.core.api.Assertions.assertThat; @@ -134,6 +132,7 @@ import org.apache.hadoop.ozone.om.KeyManagerImpl; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OMUpgradeTestUtils; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -153,7 +152,6 @@ import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; -import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.ozone.compaction.log.CompactionLogEntry; @@ -234,6 +232,7 @@ private void init() throws Exception { // Enable filesystem snapshot feature for the test regardless of the default conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion()); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS); conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, -1); conf.setTimeDuration(OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL, 100, TimeUnit.MILLISECONDS); @@ -324,23 +323,14 @@ private static void assertFinalizationException(OMException omException) { * (status FINALIZATION_DONE). */ private void finalizeOMUpgrade() throws Exception { - // Trigger OM upgrade finalization. Ref: FinalizeUpgradeSubCommand#call final OzoneManagerProtocol omClient = client.getObjectStore() .getClientProxy().getOzoneManagerClient(); - final String upgradeClientID = "Test-Upgrade-Client-" + UUID.randomUUID(); - UpgradeFinalization.StatusAndMessages finalizationResponse = - omClient.finalizeUpgrade(upgradeClientID); - - // The status should transition as soon as the client call above returns - assertTrue(isStarting(finalizationResponse.status())); - // Wait for the finalization to be marked as done. - // 10s timeout should be plenty. - await(POLL_MAX_WAIT_MILLIS, POLL_INTERVAL_MILLIS, () -> { - final UpgradeFinalization.StatusAndMessages progress = - omClient.queryUpgradeFinalizationProgress( - upgradeClientID, false, false); - return isDone(progress.status()); - }); + // TODO - OZONE_FINAL_COMMAND - change to sending command when it is ready. This will trigger OM finalization + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .addCacheEntry(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore", 1); + cluster.getOzoneManager().getMetadataManager().getMetaTable() + .put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignore"); + OMUpgradeTestUtils.waitForFinalization(omClient); } @AfterAll diff --git a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties index c5e9f8d2c7dc..d7051b6cfdae 100644 --- a/hadoop-ozone/integration-test/src/test/resources/auditlog.properties +++ b/hadoop-ozone/integration-test/src/test/resources/auditlog.properties @@ -52,7 +52,7 @@ filter.write.onMismatch = NEUTRAL # TRACE (least specific, a lot of data) # ALL (least specific, all data) -appenders = console, audit +appenders = console, audit, systemaudit appender.console.type = Console appender.console.name = STDOUT appender.console.layout.type = PatternLayout @@ -64,15 +64,27 @@ appender.audit.fileName=audit.log appender.audit.layout.type=PatternLayout appender.audit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n -loggers=audit,omSystemAudit +loggers=audit,systemaudit + logger.audit.name=OMAudit -logger.audit.level = INFO -logger.audit.appenderRefs = audit -logger.audit.appenderRef.file.ref = AUDITLOG -logger.omSystemAudit.name=OMSystemAudit -logger.omSystemAudit.level = INFO -logger.omSystemAudit.appenderRefs = audit -logger.omSystemAudit.appenderRef.file.ref = AUDITLOG +logger.audit.level=INFO +logger.audit.additivity=false +logger.audit.appenderRefs=audit,console +logger.audit.appenderRef.audit.ref=AUDITLOG +logger.audit.appenderRef.console.ref=STDOUT + +appender.systemaudit.type = File +appender.systemaudit.name = SYSTEMAUDITLOG +appender.systemaudit.fileName=system_audit.log +appender.systemaudit.layout.type=PatternLayout +appender.systemaudit.layout.pattern= %d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n + +logger.systemaudit.name=OMSystemAudit +logger.systemaudit.level=INFO +logger.systemaudit.additivity=false +logger.systemaudit.appenderRefs=systemaudit,console +logger.systemaudit.appenderRef.systemaudit.ref=SYSTEMAUDITLOG +logger.systemaudit.appenderRef.console.ref=STDOUT rootLogger.level = INFO rootLogger.appenderRefs = stdout diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f89dae3b3c0a..b17eed0bbe35 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -300,6 +300,7 @@ import org.apache.hadoop.ozone.om.service.QuotaRepairTask; import org.apache.hadoop.ozone.om.snapshot.defrag.SnapshotDefragService; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizeService; import org.apache.hadoop.ozone.om.upgrade.OMVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OzoneManagerAdminService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -463,6 +464,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private KeyProviderCryptoExtension kmsProvider; private final OMVersionManager versionManager; + private OMUpgradeFinalizeService omUpgradeFinalizeService; private final ReplicationConfigValidator replicationConfigValidator; @@ -639,6 +641,12 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) this.ozoneLockProvider = new OzoneLockProvider(getKeyPathLockEnabled(), getEnableFileSystemPaths()); + if (versionManager.needsFinalization()) { + long intervalMs = conf.getTimeDuration(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, + OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + omUpgradeFinalizeService = new OMUpgradeFinalizeService(this, versionManager, getScmClient(), intervalMs); + } + // For testing purpose only, not hit scm from om as Hadoop UGI can't login // two principals in the same JVM. ScmInfo scmInfo; @@ -1900,6 +1908,10 @@ public void start() throws IOException { bootstrap(omNodeDetails); } + if (omUpgradeFinalizeService != null) { + omUpgradeFinalizeService.start(); + } + omState = State.RUNNING; auditMap.put("NewOmState", omState.name()); SYSTEMAUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMSystemAction.STARTUP, auditMap)); @@ -2441,6 +2453,10 @@ public boolean stop() { bucketUtilizationMetrics.unRegister(); } + if (omUpgradeFinalizeService != null) { + omUpgradeFinalizeService.shutdown(); + } + if (versionManager != null) { versionManager.close(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java index 2bff57e9a8dc..0aab6e4cdc39 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/upgrade/OMFinalizeUpgradeRequest.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -59,7 +60,7 @@ public OMFinalizeUpgradeRequest(OMRequest omRequest) { @Override public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) { LOG.trace("Request: {}", getOmRequest()); - AuditLogger auditLogger = ozoneManager.getAuditLogger(); + AuditLogger auditLogger = ozoneManager.getSystemAuditLogger(); OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); OMResponse.Builder responseBuilder = OmResponseUtil.getOMResponseBuilder(getOmRequest()); @@ -95,6 +96,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut omMetadataManager.getMetaTable().addCacheEntry( new CacheKey<>(APPARENT_VERSION_KEY), CacheValue.get(context.getIndex(), String.valueOf(apparentVersion))); + // Clear the finalization_in_progress key from the cache + omMetadataManager.getMetaTable().addCacheEntry( + new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY), CacheValue.get(context.getIndex())); FinalizeUpgradeResponse omResponse = FinalizeUpgradeResponse.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java index 1d25f9096f4d..8dc981ba5eda 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -55,5 +56,7 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, APPARENT_VERSION_KEY, String.valueOf(serializedApparentVersion)); } + // Finalization has completed successfully, so the IN_PROGRESS Key should be removed from the database. + omMetadataManager.getMetaTable().deleteWithBatch(batchOperation, OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java new file mode 100644 index 000000000000..93926ba65944 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.apache.hadoop.ozone.OzoneConsts.FINALIZATION_IN_PROGRESS_KEY; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ScmClient; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.ratis.protocol.ClientId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A background service that periodically checks whether SCM has completed finalization of an upgrade and, if so, + * finalizes the OM upgrade. + */ +public class OMUpgradeFinalizeService extends BackgroundService { + + private static final Logger LOG = LoggerFactory.getLogger(OMUpgradeFinalizeService.class); + + private static final int THREAD_POOL_SIZE = 1; + private static final TimeUnit INTERVAL_UNIT = TimeUnit.MILLISECONDS; + private static final long TIMEOUT = 60000; + private static final AtomicLong RUN_COUNT = new AtomicLong(0); + + private final OzoneManager ozoneManager; + private final OMVersionManager versionManager; + private final ScmClient scmClient; + private final AtomicBoolean stopInitiated = new AtomicBoolean(false); + private final ClientId clientId = ClientId.randomId(); + + /** + * Creates an {@code OMUpgradeFinalizeService} with a custom check interval. + * + * @param ozoneManager the OzoneManager instance + * @param versionManager the {@link OMVersionManager} to query the finalization status + * @param scmClient the scmClient instance used to query SCM + * @param intervalMs the duration to wait between checks + */ + public OMUpgradeFinalizeService(OzoneManager ozoneManager, OMVersionManager versionManager, ScmClient scmClient, + long intervalMs) { + super("OMUpgradeFinalizeService", intervalMs, INTERVAL_UNIT, THREAD_POOL_SIZE, TIMEOUT, + ozoneManager.getThreadNamePrefix()); + this.ozoneManager = ozoneManager; + this.versionManager = versionManager; + this.scmClient = scmClient; + } + + @Override + public BackgroundTaskQueue getTasks() { + BackgroundTaskQueue queue = new BackgroundTaskQueue(); + if (!versionManager.needsFinalization()) { + // Finalization is done (or was never needed), so this service can now shutdown. To avoid deadlocking on the + // executor.awaitTermination by calling shutdown directly, spawn a thread to perform the shutdown which will + // block until this task / thread completes in the executor. + if (stopInitiated.compareAndSet(false, true)) { + LOG.info("OMUpgradeFinalizeService: finalization is no longer needed, shutting down."); + Thread stopper = new Thread(this::shutdown, "OMUpgradeFinalizeService-stopper"); + stopper.setDaemon(true); + stopper.start(); + } + return queue; // empty — PeriodicalTask.run() will return without scheduling work + } + if (ozoneManager.isLeaderReady()) { + queue.add(new UpgradeStatusCheckTask()); + } + return queue; + } + + /** + * Periodic task that checks upgrade finalization status and logs the result. + */ + private class UpgradeStatusCheckTask implements BackgroundTask { + + @Override + public BackgroundTaskResult call() { + final long run = RUN_COUNT.incrementAndGet(); + if (!ozoneManager.isLeaderReady()) { + LOG.debug("OMUpgradeFinalizeService: skipping check — not the leader. Run count {}", run); + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + if (versionManager.needsFinalization()) { + try { + // To finalize OM, first finalization needs to have been started. Then SCM needs to indicate that it has + // completed its finalization work. Only once both of those things have happened can OM finalize. + String finalizationInProgress = + ozoneManager.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY); + if (finalizationInProgress == null) { + LOG.debug("OMUpgradeFinalizeService: skipping check — finalization is not in progress. Run count {}", run); + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + + HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); + if (upgradeStatus.getShouldFinalize()) { + LOG.info("The SCM Upgrade has been finalized. OM will now finalize. Run count {}", run); + + OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) + .setClientId(clientId.toString()) + .build(); + OzoneManagerProtocolProtos.OMResponse response = OzoneManagerRatisUtils.submitRequest( + ozoneManager, omRequest, clientId, run); + if (!response.getSuccess()) { + LOG.error("Failed to send FinalizeUpgradeRequest to over Ratis. {}. Run count {}", + response.getMessage(), run); + } + } else { + LOG.debug("The SCM Upgrade has not been finalized. Run count {}", run); + } + } catch (Exception e) { + LOG.error("An exception occurred while trying to check the SCM Upgrade status or finalize OM. Run count {}", + run, e); + } + } else { + LOG.debug("Finalization is not in progress. Run count {}", run); + } + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java new file mode 100644 index 000000000000..863ead019ed5 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/upgrade/TestOMFinalizeUpgradeRequest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.upgrade; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Collections; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; +import org.apache.hadoop.ozone.om.request.key.TestOMKeyRequest; +import org.apache.hadoop.ozone.om.upgrade.OMVersionManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.server.protocol.TermIndex; +import org.junit.jupiter.api.Test; + +/** + * Tests for the OMFinalizeUpgradeRequest class. + */ +public class TestOMFinalizeUpgradeRequest extends TestOMKeyRequest { + + @Test + public void testFinalizationInProgressKeyRemoved() throws IOException { + OMVersionManager omVersionManager = mock(OMVersionManager.class); + when(omVersionManager.getApparentVersion()).thenReturn(OzoneManagerVersion.DEFAULT_VERSION); + when(ozoneManager.getVersionManager()).thenReturn(omVersionManager); + when(ozoneManager.finalizeUpgrade(any())).thenReturn(new UpgradeFinalization.StatusAndMessages( + UpgradeFinalization.Status.FINALIZATION_IN_PROGRESS, Collections.singletonList("Finalization in progress"))); + + omMetadataManager.getMetaTable().put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignored"); + omMetadataManager.getMetaTable().addCacheEntry( + new CacheKey<>(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY), CacheValue.get(1, "ignored")); + + String progressKey = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + + assertNotNull(progressKey); + submitRequest(); + + progressKey = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + assertNull(progressKey); + } + + private void submitRequest() throws IOException { + OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) + .setClientId(ClientId.randomId().toString()) + .build(); + + OMFinalizeUpgradeRequest request = new OMFinalizeUpgradeRequest(omRequest); + ExecutionContext context = ExecutionContext.of(1, TermIndex.INITIAL_VALUE); + + OzoneManagerProtocolProtos.OMRequest modifiedOmRequest = request.preExecute(ozoneManager); + + // Will not be equal, as UserInfo will be set. + assertNotEquals(omRequest, modifiedOmRequest); + request.validateAndUpdateCache(ozoneManager, context); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java new file mode 100644 index 000000000000..15ad3c848b5f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/TestOMFinalizeUpgradeResponse.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.upgrade; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.IOException; +import java.nio.file.Path; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for the OMFinalizeUpgradeResponse class. + */ +public class TestOMFinalizeUpgradeResponse { + + @TempDir + private Path folder; + private OMMetadataManager omMetadataManager; + private BatchOperation batchOperation; + + @BeforeEach + public void setup() throws Exception { + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, + folder.toAbsolutePath().toString()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); + batchOperation = omMetadataManager.getStore().initBatchOperation(); + } + + @Test + public void testFinalizationInProgressKeyRemoved() throws IOException { + // Add the in progress key which would normally have been added by the start finalization command + omMetadataManager.getMetaTable().put(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY, "ignored"); + + String value = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + assertEquals("ignored", value); + + OMFinalizeUpgradeResponse finalizeUpgradeResponse = new OMFinalizeUpgradeResponse(createRequest(), 1); + finalizeUpgradeResponse.addToDBBatch(omMetadataManager, batchOperation); + + omMetadataManager.getStore().commitBatchOperation(batchOperation); + + // Ensure the key is removed as expected + value = omMetadataManager.getMetaTable().get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY); + assertNull(value); + } + + private OzoneManagerProtocolProtos.OMResponse createRequest() { + return OzoneManagerProtocolProtos.OMResponse.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.FinalizeUpgrade) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .build(); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java new file mode 100644 index 000000000000..ef001e338eb9 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/upgrade/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package contains test classes for upgrade responses. + */ +package org.apache.hadoop.ozone.om.response.upgrade; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java new file mode 100644 index 000000000000..a3aa7be8b5f0 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.google.protobuf.ServiceException; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.utils.db.CodecException; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; +import org.apache.hadoop.hdds.utils.db.TypedTable; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ScmClient; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.ratis.protocol.ClientId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for {@link OMUpgradeFinalizeService}. + * Uses {@link org.apache.hadoop.hdds.utils.BackgroundService#runPeriodicalTaskNow()} to execute + * tasks synchronously on the test thread, avoiding timing-dependent polling. + */ +public class TestOMUpgradeFinalizeService { + + // A long interval so the scheduler never fires automatically during tests; + // we drive execution manually via runPeriodicalTaskNow(). + private static final long INTERVAL_MS = 60_000; + + private OzoneManager ozoneManager; + private OMVersionManager versionManager; + private TypedTable metaTable; + private ScmClient scmClient; + private StorageContainerLocationProtocol containerClient; + private OzoneManagerRatisServer omRatisServer; + private OMUpgradeFinalizeService service; + + @BeforeEach + void setUp() throws RocksDatabaseException, CodecException { + ozoneManager = mock(OzoneManager.class); + OMMetadataManager metadataManager = mock(OMMetadataManager.class); + metaTable = mock(TypedTable.class); + when(ozoneManager.getThreadNamePrefix()).thenReturn(""); + when(ozoneManager.getOMNodeId()).thenReturn("clientId"); + when(ozoneManager.getMetadataManager()).thenReturn(metadataManager); + when(metadataManager.getMetaTable()).thenReturn(metaTable); + // For most tests, set the finalization command as having been received + when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn("ignored"); + + versionManager = mock(OMVersionManager.class); + // preExecute() calls ozoneManager.getVersionManager().getApparentVersion().serialize() + when(ozoneManager.getVersionManager()).thenReturn(versionManager); + when(versionManager.getApparentVersion()).thenReturn(OzoneManagerVersion.DEFAULT_VERSION); + + omRatisServer = mock(OzoneManagerRatisServer.class); + // OzoneManagerRatisUtils.submitRequest() calls ozoneManager.getOmRatisServer().submitRequest(...) + when(ozoneManager.getOmRatisServer()).thenReturn(omRatisServer); + + containerClient = mock(StorageContainerLocationProtocol.class); + scmClient = mock(ScmClient.class); + when(scmClient.getContainerClient()).thenReturn(containerClient); + + service = new OMUpgradeFinalizeService(ozoneManager, versionManager, scmClient, INTERVAL_MS); + } + + /** + * When the OM is not the leader, getTasks() should return an empty queue + * and no interaction with the SCM client or Ratis server should occur. + */ + @Test + void testNoTasksSubmittedWhenNotLeader() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(false); + when(versionManager.needsFinalization()).thenReturn(true); + + service.runPeriodicalTaskNow(); + + verifyNoInteractions(scmClient); + verifyNoInteractions(omRatisServer); + } + + /** + * When finalization is not needed, getTasks() should return an empty queue + * and no SCM query or Ratis submission should occur. + */ + @Test + void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(false); + + service.runPeriodicalTaskNow(); + + verifyNoInteractions(scmClient); + verifyNoInteractions(omRatisServer); + } + + /** + * When the OM is the leader, finalization is needed, the finalization command is given and SCM reports + * shouldFinalize=true, a FinalizeUpgrade request should be submitted via Ratis. + */ + @Test + void testFinalizationTriggeredWhenScmIsFinalizedAndFinalizationInProgress() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + + HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(true) + .setShouldFinalize(true) + .setNumDatanodesFinalized(3) + .setNumDatanodesTotal(3) + .build(); + when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); + // Finalization command not given yet + when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn(null); + + service.runPeriodicalTaskNow(); + + verifyNoInteractions(containerClient); + verifyNoInteractions(omRatisServer); + + when(metaTable.get(OzoneConsts.FINALIZATION_IN_PROGRESS_KEY)).thenReturn("ignored"); + service.runPeriodicalTaskNow(); + + verify(containerClient).queryUpgradeStatus(); + // Implementation submits a FinalizeUpgrade request through Ratis + verify(omRatisServer).submitRequest(any(), any(ClientId.class), anyLong()); + } + + /** + * When SCM reports shouldFinalize=false (SCM is not yet finalized), + * no Ratis request should be submitted. + */ + @Test + void testFinalizationSkippedWhenScmNotYetFinalized() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + + HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(false) + .setShouldFinalize(false) + .setNumDatanodesFinalized(0) + .setNumDatanodesTotal(3) + .build(); + when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); + + service.runPeriodicalTaskNow(); + + verify(containerClient).queryUpgradeStatus(); + verifyNoInteractions(omRatisServer); + } + + /** + * When the SCM client throws an IOException, the service should absorb it + * and not submit any Ratis request. + */ + @Test + void testExceptionFromScmClientIsHandledGracefully() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + when(containerClient.queryUpgradeStatus()).thenThrow(new IOException("SCM unavailable")); + + // The catch block in the task swallows the exception. + service.runPeriodicalTaskNow(); + + verify(containerClient).queryUpgradeStatus(); + verifyNoInteractions(omRatisServer); + } + + /** + * When {@code needsFinalization()} returns {@code false}, {@link OMUpgradeFinalizeService#getTasks()} + * should spawn a stopper thread that calls {@link OMUpgradeFinalizeService#shutdown()} exactly once, + * even when {@code getTasks()} is driven multiple times (guarded by the internal + * {@code stopInitiated} AtomicBoolean). No SCM or Ratis interactions should occur. + *

+ * The service is subclassed to intercept {@code shutdown()} via a {@link CountDownLatch}, + * avoiding both actual executor teardown and any need for {@code Thread.sleep}. + */ + @Test + void testShutdownTriggeredExactlyOnceWhenFinalizationNoLongerNeeded() throws Exception { + AtomicInteger shutdownCount = new AtomicInteger(0); + CountDownLatch firstShutdown = new CountDownLatch(1); + + OMUpgradeFinalizeService testService = new OMUpgradeFinalizeService( + ozoneManager, versionManager, scmClient, INTERVAL_MS) { + @Override + public synchronized void shutdown() { + shutdownCount.incrementAndGet(); + firstShutdown.countDown(); + // Don't propagate to super — avoids racing on the test executor. + } + }; + + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(false); + + // Drive getTasks() three times; the stopper thread should only fire once. + testService.runPeriodicalTaskNow(); + testService.runPeriodicalTaskNow(); + testService.runPeriodicalTaskNow(); + + // Wait for the single stopper thread to call shutdown(). + assertTrue(firstShutdown.await(5, TimeUnit.SECONDS), + "shutdown() should have been called by the stopper thread within 5 s"); + // incrementAndGet() happens before countDown(), so the count is already final — no sleep needed. + assertEquals(1, shutdownCount.get(), + "shutdown() must be called exactly once despite multiple getTasks() invocations"); + verifyNoInteractions(scmClient); + verifyNoInteractions(omRatisServer); + } + + /** + * When the Ratis submission throws, the service should absorb the error + * and not propagate the exception. + */ + @Test + void testExceptionFromRatisSubmitIsHandledGracefully() throws Exception { + when(ozoneManager.isLeaderReady()).thenReturn(true); + when(versionManager.needsFinalization()).thenReturn(true); + + HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalized(true) + .setShouldFinalize(true) + .setNumDatanodesFinalized(3) + .setNumDatanodesTotal(3) + .build(); + when(containerClient.queryUpgradeStatus()).thenReturn(scmStatus); + when(omRatisServer.submitRequest(any(), any(ClientId.class), anyLong())) + .thenThrow(new ServiceException("Ratis unavailable")); + + // The catch block in the task swallows the exception. + service.runPeriodicalTaskNow(); + + // submitRequest was attempted but threw — service should not propagate the exception. + verify(omRatisServer).submitRequest(any(), any(ClientId.class), anyLong()); + } +}