diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java new file mode 100644 index 000000000000..bf2a8543d165 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentUpgradeActionProvider.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import java.util.Map; +import org.apache.hadoop.hdds.ComponentVersion; + +/** + * Supplies upgrade actions keyed by {@link ComponentVersion}. Implementations typically perform classpath scanning or + * return a fixed map for tests. The component version manager decides when each action is invoked. + * + * @param concrete upgrade action type (for example OM-specific or HDDS-specific) + */ +@FunctionalInterface +public interface ComponentUpgradeActionProvider { + + /** + * Returns all upgrade actions from this provider, keyed by component version. + *

+ * Implementations must return a newly allocated map on each call; the caller may retain and use it directly. + */ + Map load(); +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java index 20a7d02d178f..ebdd7d730037 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeException.java @@ -106,10 +106,8 @@ public String toString() { public enum ResultCodes { OK, INVALID_REQUEST, - UPDATE_LAYOUT_VERSION_FAILED, - LAYOUT_FEATURE_FINALIZATION_FAILED, - PREFINALIZE_ACTION_VALIDATION_FAILED, - FIRST_UPGRADE_START_ACTION_FAILED, - PREFINALIZE_VALIDATION_FAILED; + APPARENT_VERSION_UPDATE_FAILED, + UPGRADE_FINALIZATION_FAILED, + FINALIZE_UPGRADE_ACTION_FAILED, } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalization.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalization.java index 218a09b6edc5..846ff28bdb4f 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalization.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeFinalization.java @@ -49,12 +49,19 @@ public final class UpgradeFinalization { ); /** - * Default message to provide when the service is in ALREADY_FINALIZED state. + * Returned from finalize upgrade commands when finalization is not required. */ public static final StatusAndMessages FINALIZED_MSG = new StatusAndMessages( Status.ALREADY_FINALIZED, Collections.emptyList() ); + /** + * Returned from progress/status queries when finalization is not required. + */ + public static final StatusAndMessages FINALIZATION_DONE_MSG = new StatusAndMessages( + Status.FINALIZATION_DONE, Collections.emptyList() + ); + /** * Represents the current state in which the service is with regards to * finalization after an upgrade. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java index 07bdedb4398e..d45623f36bb1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java @@ -261,7 +261,7 @@ private static DatanodeDetailsYaml getDatanodeDetailsYaml( HDDSLayoutFeature layoutFeature = f.getAnnotation(BelongsToHDDSLayoutVersion.class).value(); if (layoutFeature.layoutVersion() > - datanodeLayoutStorage.getLayoutVersion()) { + datanodeLayoutStorage.getApparentVersion()) { continue; } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 2f53178e9bf9..5263b7b87a66 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -168,7 +168,7 @@ public DatanodeStateMachine(HddsDatanodeService hddsDatanodeService, datanodeDetails.getUuidString()); layoutVersionManager = new HDDSLayoutVersionManager( - layoutStorage.getLayoutVersion()); + layoutStorage.getApparentVersion()); upgradeFinalizer = new DataNodeUpgradeFinalizer(layoutVersionManager); VersionedDatanodeFeatures.initialize(layoutVersionManager); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java index 769d556643cd..7fd352ac0f27 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DataNodeUpgradeFinalizer.java @@ -47,7 +47,7 @@ public void finalizeLayoutFeature(LayoutFeature layoutFeature, String msg = String.format("Failed to finalize datanode layout feature " + "%s. It is not an HDDS Layout Feature.", layoutFeature); throw new UpgradeException(msg, - UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); + UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED); } } } diff --git a/hadoop-hdds/docs/content/design/upgrade-dev-primer.md b/hadoop-hdds/docs/content/design/upgrade-dev-primer.md index 13f6f5a683d7..8934e412f8a5 100644 --- a/hadoop-hdds/docs/content/design/upgrade-dev-primer.md +++ b/hadoop-hdds/docs/content/design/upgrade-dev-primer.md @@ -36,13 +36,11 @@ Downgrade to a lower version is allowed from the pre-finalized state. This invol org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature Class to add a new layout feature being brought in. Layout version is typically 1 + last layout feature in that catalog. -## LayoutVersionManager - org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager - org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager -Every component carries an instance of this interface, which provides APIs to get runtime layout version, and if a feature is allowed based on that or not. +## Version management (OM vs HDDS) -The LayoutVersionManager interface carries an API that can be used to check if a feature is allowed in the current layout version. - org.apache.hadoop.ozone.upgrade.LayoutVersionManager#isAllowed(org.apache.hadoop.ozone.upgrade.LayoutFeature) +**Ozone Manager** uses [`org.apache.hadoop.ozone.om.upgrade.OMVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.ComponentVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java)), with upgrade actions discovered via [`org.apache.hadoop.ozone.om.upgrade.OMUpgradeActionProvider`](https://github.com/apache/ozone/blob/master/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java). It exposes apparent/software `ComponentVersion` and `isAllowed(ComponentVersion)` for gating. + +**SCM / DataNode** continue to use [`org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutVersionManager.java) ([`org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager`](https://github.com/apache/ozone/blob/master/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java)), which provides metadata/software layout integers and `isAllowed(LayoutFeature)`. ## @DisallowedUntilLayoutVersion Annotation Method level annotation used to "disallow" an API if current layout version does not include the associated layout feature. Currently it is added only to the OM module, but can easily be moved down to a common module based on need on the HDDS layer. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java index ad6bad1c7029..9498859067ca 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSVersionManager.java @@ -20,29 +20,45 @@ import java.io.IOException; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.apache.hadoop.ozone.upgrade.UpgradeException; /** * Component version manager for HDDS (Datanodes and SCM). */ public class HDDSVersionManager extends ComponentVersionManager { - public HDDSVersionManager(int serializedApparentVersion) throws IOException { - super(computeApparentVersion(serializedApparentVersion), HDDSVersion.SOFTWARE_VERSION); + public HDDSVersionManager(Storage storage) throws IOException { + super(storage, computeApparentVersion(storage.getApparentVersion()), HDDSVersion.SOFTWARE_VERSION); } /** - * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the - * ZDU feature, and the apparent version corresponds to a version in {@link HDDSVersion}. - * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the - * ZDU feature, and the apparent version corresponds to a version in {@link HDDSLayoutFeature}. + * If the apparent version stored on the disk is >= {@link HDDSVersion#ZDU} serialized, the apparent version is + * resolved via {@link HDDSVersion#deserialize(int)}. Values with no matching {@link HDDSVersion} fail startup with + * the persisted integer in the exception message. + * If the value is below that threshold, the apparent version is resolved as a {@link HDDSLayoutFeature}. Integers in + * the gap between the largest {@link HDDSLayoutFeature} and ZDU are not valid legacy layout values; startup fails + * with the persisted integer in the exception message. */ - private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { - if (serializedApparentVersion < HDDSVersion.ZDU.serialize()) { - return HDDSLayoutFeature.deserialize(serializedApparentVersion); + private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException { + if (serializedApparentVersion >= HDDSVersion.ZDU.serialize()) { + HDDSVersion fromHdds = HDDSVersion.deserialize(serializedApparentVersion); + if (fromHdds != HDDSVersion.FUTURE_VERSION) { + return fromHdds; + } } else { - return HDDSVersion.deserialize(serializedApparentVersion); + ComponentVersion fromLayout = HDDSLayoutFeature.deserialize(serializedApparentVersion); + if (fromLayout != null) { + return fromLayout; + } } + throw new IOException("Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion + + " for software version " + HDDSVersion.SOFTWARE_VERSION + ". Make sure this component was not downgraded" + + " after finalization"); } - // TODO HDDS-14826: Register upgrade actions based on annotations + @Override + protected void runUpgradeAction(ComponentVersion componentVersion) throws UpgradeException { + // TODO HDDS-14826: Register upgrade actions based on annotations + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/Storage.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/Storage.java index ae357d8d7519..4b0c49b02445 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/Storage.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/Storage.java @@ -124,20 +124,12 @@ public void setClusterId(String clusterId) throws IOException { } } - public int getLayoutVersion() { - return storageInfo.getLayoutVersion(); + public int getApparentVersion() { + return storageInfo.getApparentVersion(); } - public void setLayoutVersion(int version) { - storageInfo.setLayoutVersion(version); - } - - public void setFirstUpgradeActionLayoutVersion(int version) { - storageInfo.setFirstUpgradeActionLayoutVersion(version); - } - - public int getFirstUpgradeActionLayoutVersion() { - return storageInfo.getFirstUpgradeActionLayoutVersion(); + public void setApparentVersion(int version) { + storageInfo.setApparentVersion(version); } /** @@ -288,8 +280,7 @@ public void persistCurrentState() throws IOException { storageInfo.writeTo(getVersionFile()); } - protected static int getInitLayoutVersion(OzoneConfiguration conf, - String configKey, + protected static int getInitApparentVersion(OzoneConfiguration conf, String configKey, IntSupplier defaultLvSupplier) { int lV = conf.getInt(configKey, OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT); if (lV == OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java index 7a74258936fe..624b1affde7d 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java @@ -61,9 +61,6 @@ public class StorageInfo { */ private static final String LAYOUT_VERSION = "layoutVersion"; - private static final String FIRST_UPGRADE_ACTION_LAYOUT_VERSION = - "firstUpgradeActionLayoutVersion"; - private static final int INVALID_LAYOUT_VERSION = -1; /** @@ -93,7 +90,7 @@ public StorageInfo(NodeType type, File propertiesFile) verifyNodeType(type); verifyClusterId(); verifyCreationTime(); - verifyLayoutVersion(); + verifyApparentVersion(); } public NodeType getNodeType() { @@ -112,7 +109,7 @@ public Long getCreationTime() { return null; } - public int getLayoutVersion() { + public int getApparentVersion() { String layout = properties.getProperty(LAYOUT_VERSION); if (layout != null) { return Integer.parseInt(layout); @@ -120,7 +117,7 @@ public int getLayoutVersion() { return 0; } - private void verifyLayoutVersion() { + private void verifyApparentVersion() { String layout = getProperty(LAYOUT_VERSION); if (layout == null) { LOG.warn("Found " + STORAGE_FILE_VERSION + " file without any layout " + @@ -129,20 +126,6 @@ private void verifyLayoutVersion() { } } - public int getFirstUpgradeActionLayoutVersion() { - String upgradingTo = - properties.getProperty(FIRST_UPGRADE_ACTION_LAYOUT_VERSION); - if (upgradingTo != null) { - return Integer.parseInt(upgradingTo); - } - return INVALID_LAYOUT_VERSION; - } - - public void setFirstUpgradeActionLayoutVersion(int layoutVersion) { - properties.setProperty( - FIRST_UPGRADE_ACTION_LAYOUT_VERSION, Integer.toString(layoutVersion)); - } - public String getProperty(String key) { return properties.getProperty(key); } @@ -159,7 +142,7 @@ public void setClusterId(String clusterId) { properties.setProperty(CLUSTER_ID, clusterId); } - public void setLayoutVersion(int version) { + public void setApparentVersion(int version) { properties.setProperty(LAYOUT_VERSION, Integer.toString(version)); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java index fda69ff872f0..8d0a31a8a4ac 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/BasicUpgradeFinalizer.java @@ -17,9 +17,9 @@ package org.apache.hadoop.ozone.upgrade; +import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.APPARENT_VERSION_UPDATE_FAILED; import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.INVALID_REQUEST; -import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED; -import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.UPDATE_LAYOUT_VERSION_FAILED; +import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZATION_IN_PROGRESS_MSG; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZATION_REQUIRED_MSG; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.FINALIZED_MSG; @@ -291,11 +291,11 @@ protected void updateLayoutVersionInVersionFile(LayoutFeature feature, } private int currentStoredLayoutVersion(Storage config) { - return config.getLayoutVersion(); + return config.getApparentVersion(); } private void updateStorageLayoutVersion(int version, Storage config) { - config.setLayoutVersion(version); + config.setApparentVersion(version); } private void persistStorage(Storage config) throws IOException { @@ -326,13 +326,13 @@ protected void logAndEmit(String msg) { protected void logFinalizationFailureAndThrow(Exception e, String feature) throws UpgradeException { String msg = "Error during finalization of " + feature + "."; - logAndThrow(e, msg, LAYOUT_FEATURE_FINALIZATION_FAILED); + logAndThrow(e, msg, UPGRADE_FINALIZATION_FAILED); } private void logLayoutVersionUpdateFailureAndThrow(IOException e) throws UpgradeException { String msg = "Updating the LayoutVersion in the VERSION file failed."; - logAndThrow(e, msg, UPDATE_LAYOUT_VERSION_FAILED); + logAndThrow(e, msg, APPARENT_VERSION_UPDATE_FAILED); } private void logAndThrow(Exception e, String msg, ResultCodes resultCode) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java index 11bf4a3f2281..b2f6c7e6c615 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/ComponentVersionManager.java @@ -17,11 +17,15 @@ package org.apache.hadoop.ozone.upgrade; +import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.APPARENT_VERSION_UPDATE_FAILED; + +import com.google.common.annotations.VisibleForTesting; import java.io.Closeable; import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.common.Storage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,22 +55,16 @@ public abstract class ComponentVersionManager implements Closeable { // Software version will never change. private final ComponentVersion softwareVersion; private final ComponentVersionManagerMetrics metrics; + private final Storage storage; - private static final Logger LOG = - LoggerFactory.getLogger(ComponentVersionManager.class); + private static final Logger LOG = LoggerFactory.getLogger(ComponentVersionManager.class); - protected ComponentVersionManager(ComponentVersion apparentVersion, ComponentVersion softwareVersion) - throws IOException { + protected ComponentVersionManager(Storage storage, ComponentVersion apparentVersion, + ComponentVersion softwareVersion) { + this.storage = storage; this.apparentVersion = apparentVersion; this.softwareVersion = softwareVersion; - if (!apparentVersion.isSupportedBy(softwareVersion)) { - throw new IOException( - "Cannot initialize ComponentVersionManager. Apparent version " - + apparentVersion + " is larger than software version " - + softwareVersion); - } - LOG.info("Initializing version manager with apparent version {} and software version {}", apparentVersion, softwareVersion); this.metrics = ComponentVersionManagerMetrics.create(this); @@ -88,11 +86,30 @@ public boolean needsFinalization() { return !apparentVersion.equals(softwareVersion); } + /** + * Test-only accessor for the {@link Storage} instance supplied to the constructor. + */ + @VisibleForTesting + protected Storage getStorageForTesting() { + return storage; + } + + public void finalizeUpgrade() throws UpgradeException { + for (ComponentVersion version : getUnfinalizedVersions()) { + validateForFinalization(version); + runUpgradeAction(version); + persistApparentVersion(version); + + LOG.info("Version {} has been finalized.", version); + } + LOG.info("Finalization is complete."); + } + /** * @return An Iterable of all versions after the current apparent version which still need to be finalized. If this * component is already finalized, the Iterable will be empty. */ - public Iterable getUnfinalizedVersions() { + private Iterable getUnfinalizedVersions() { return () -> new Iterator() { private ComponentVersion currentVersion = apparentVersion; @@ -118,7 +135,7 @@ public ComponentVersion next() { * * @param newApparentVersion The version to mark as finalized. */ - public void markFinalized(ComponentVersion newApparentVersion) { + private void validateForFinalization(ComponentVersion newApparentVersion) { String versionMsg = "Software version: " + softwareVersion + ", apparent version: " + apparentVersion + ", provided version: " + newApparentVersion @@ -131,19 +148,34 @@ public void markFinalized(ComponentVersion newApparentVersion) { ComponentVersion nextVersion = apparentVersion.nextVersion(); if (nextVersion == null) { throw new IllegalArgumentException("Attempt to finalize when no future versions exist." + versionMsg); - } else if (nextVersion.equals(newApparentVersion)) { - apparentVersion = newApparentVersion; - LOG.info("Version {} has been finalized.", apparentVersion); - if (!needsFinalization()) { - LOG.info("Finalization is complete."); - } - } else { + } else if (newApparentVersion != nextVersion) { throw new IllegalArgumentException( - "Finalize attempt on a version that is newer than the next feature to be finalized. " + versionMsg); + "Finalize attempt on a version that is not the next feature to be finalized. " + versionMsg); } } } + private void persistApparentVersion(ComponentVersion version) throws UpgradeException { + int prevVersion = storage.getApparentVersion(); + + storage.setApparentVersion(version.serialize()); + try { + storage.persistCurrentState(); + } catch (IOException e) { + storage.setApparentVersion(prevVersion); + logAndThrow(e, "Updating version in the VERSION file from " + prevVersion + " to " + version + + " failed.", APPARENT_VERSION_UPDATE_FAILED); + } + apparentVersion = version; + } + + protected void logAndThrow(Exception e, String msg, UpgradeException.ResultCodes resultCode) throws UpgradeException { + LOG.error(msg, e); + throw new UpgradeException(msg, e, resultCode); + } + + protected abstract void runUpgradeAction(ComponentVersion componentVersion) throws UpgradeException; + @Override public void close() { metrics.unRegister(); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java index 1e76ff91be39..77a3720a25b2 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSVersionManager.java @@ -17,6 +17,8 @@ package org.apache.hadoop.hdds.upgrade; +import static org.mockito.Mockito.when; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -24,12 +26,17 @@ import java.util.stream.Stream; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.HDDSVersion; +import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest; import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.apache.ozone.test.tag.Unhealthy; +import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; /** - * Tests for {@link HDDSVersionManager}. + * Tests for {@link HDDSVersionManager} using on-disk {@link Storage} under a JUnit temp directory. */ class TestHDDSVersionManager extends AbstractComponentVersionManagerTest { @@ -52,9 +59,20 @@ public static Stream preFinalizedVersionArgs() { .map(Arguments::of); } + @ParameterizedTest + @MethodSource("preFinalizedVersionArgs") + @Override + @Unhealthy + public void testFinalizationFromEarlierVersions(ComponentVersion apparentVersion) { + // TODO Once HDDSVersionManager implementation is finished, this override can be removed to enable the test in the + // parent class. + } + @Override protected ComponentVersionManager createManager(int serializedApparentVersion) throws IOException { - return new HDDSVersionManager(serializedApparentVersion); + Storage mockStorage = Mockito.mock(Storage.class); + when(mockStorage.getApparentVersion()).thenReturn(serializedApparentVersion); + return new HDDSVersionManager(mockStorage); } @Override diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java index 0bf439326cac..b5a38623faf6 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/AbstractComponentVersionManagerTest.java @@ -23,27 +23,33 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; import java.io.IOException; -import java.util.Iterator; import java.util.List; -import java.util.NoSuchElementException; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.ozone.common.Storage; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mockito; /** * Shared tests for concrete {@link ComponentVersionManager} implementations. + * + *

Each subclass {@linkplain #createManager(int) builds} the version manager with a real {@link Storage} + * instance rooted under a JUnit temporary directory (see for example {@code TestOMStorage} in ozone-manager). + * Assertions use {@link ComponentVersionManager#getStorageForTesting()} and {@link Storage#getApparentVersion()} + * to confirm what was persisted, instead of Mockito interaction verification. */ public abstract class AbstractComponentVersionManagerTest { + /** + * Creates a new manager for {@code serializedApparentVersion}. The implementation must initialize real + * {@link Storage} on disk with that apparent version (and return a manager whose + * {@link ComponentVersionManager#getStorageForTesting()} is that instance). + */ protected abstract ComponentVersionManager createManager(int serializedApparentVersion) throws IOException; protected abstract List allVersionsInOrder(); @@ -71,65 +77,49 @@ public void testApparentVersionBehindSoftwareVersion() { } @ParameterizedTest - // Child classes must implement this as a static method to provide the versions to start finalization from. @MethodSource("preFinalizedVersionArgs") public void testFinalizationFromEarlierVersions(ComponentVersion apparentVersion) throws Exception { List allVersions = allVersionsInOrder(); int apparentVersionIndex = allVersions.indexOf(apparentVersion); assertTrue(apparentVersionIndex >= 0, "Apparent version " + apparentVersion + " must exist"); - Iterator expectedVersions = allVersions.subList(apparentVersionIndex + 1, allVersions.size()) - .iterator(); + List expectedChain = allVersions.subList(apparentVersionIndex + 1, allVersions.size()); try (ComponentVersionManager versionManager = createManager(apparentVersion.serialize())) { assertApparentVersion(versionManager, apparentVersion); - for (ComponentVersion versionToFinalize : versionManager.getUnfinalizedVersions()) { + if (!expectedChain.isEmpty()) { assertTrue(versionManager.needsFinalization()); - assertFalse(versionManager.isAllowed(versionToFinalize), - "Unfinalized version " + versionToFinalize + " should not be allowed by apparent version " - + versionManager.getApparentVersion()); - assertTrue(expectedVersions.hasNext()); - assertEquals(expectedVersions.next(), versionToFinalize); - - versionManager.markFinalized(versionToFinalize); - assertApparentVersion(versionManager, versionToFinalize); + for (ComponentVersion v : expectedChain) { + assertFalse(versionManager.isAllowed(v), + "Version " + v + " should not be allowed before finalization"); + } } - assertFalse(expectedVersions.hasNext()); - assertThrows(NoSuchElementException.class, expectedVersions::next); - } - } + versionManager.finalizeUpgrade(); - @Test - public void testFinalizationFromSoftwareVersionNoOp() throws Exception { - try (ComponentVersionManager versionManager = createManager(expectedSoftwareVersion().serialize())) { assertApparentVersion(versionManager, expectedSoftwareVersion()); assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); - - versionManager.markFinalized(expectedSoftwareVersion()); - assertApparentVersion(versionManager, expectedSoftwareVersion()); - assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + Storage storage = versionManager.getStorageForTesting(); + assertEquals(expectedSoftwareVersion().serialize(), storage.getApparentVersion(), + "Storage apparent version should match software version after finalization"); } } @Test - public void testFinalizationOfNonExistentVersion() throws Exception { + public void testFinalizationFromSoftwareVersionNoOp() throws Exception { try (ComponentVersionManager versionManager = createManager(expectedSoftwareVersion().serialize())) { assertApparentVersion(versionManager, expectedSoftwareVersion()); assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); - ComponentVersion mockVersion = Mockito.mock(ComponentVersion.class); - when(mockVersion.isSupportedBy(any())).thenReturn(false); + Storage storage = versionManager.getStorageForTesting(); + int apparentOnStorageBefore = storage.getApparentVersion(); + versionManager.finalizeUpgrade(); - assertThrows(IllegalArgumentException.class, () -> versionManager.markFinalized(mockVersion)); - // The failed finalization call should not have changed the version manager's state. assertApparentVersion(versionManager, expectedSoftwareVersion()); assertFalse(versionManager.needsFinalization()); - assertFalse(versionManager.getUnfinalizedVersions().iterator().hasNext()); + assertEquals(apparentOnStorageBefore, storage.getApparentVersion(), + "No-op finalize should not change the persisted apparent version"); } } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestBasicUpgradeFinalizer.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestBasicUpgradeFinalizer.java index 00ac2411fe11..29cac62fbb20 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestBasicUpgradeFinalizer.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/ozone/upgrade/TestBasicUpgradeFinalizer.java @@ -95,6 +95,8 @@ public void testAlreadyFinalizedDoesNotTriggerNewFinalization() StatusAndMessages res = finalizer.finalize("test-client-1", mockObj); + // TODO this test class will be removed with BasicUpgradeFinalizer when all components are migrated to the new + // version framework. assertEquals(ALREADY_FINALIZED, res.status()); assertFalse(finalizer.preCalled || finalizer.finalizeCalled || finalizer.postCalled); @@ -222,12 +224,12 @@ public void finalizeLayoutFeature(LayoutFeature lf, Object service) super.finalizeLayoutFeature(lf, lf.action(), mockStorage); - inOrder.verify(mockStorage).setLayoutVersion(eq(lf.layoutVersion())); + inOrder.verify(mockStorage).setApparentVersion(eq(lf.layoutVersion())); try { inOrder.verify(mockStorage).persistCurrentState(); } catch (IOException ex) { throw new UpgradeException(ex, - UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); + UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED); } finalizeCalled = true; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java index 9bb8809055e8..d41537fed0fb 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java @@ -48,7 +48,7 @@ public class SCMStorageConfig extends Storage { */ public SCMStorageConfig(OzoneConfiguration conf) throws IOException { super(NodeType.SCM, ServerUtils.getScmDbDir(conf), STORAGE_DIR, - getInitLayoutVersion(conf, TESTING_INIT_LAYOUT_VERSION_KEY, + getInitApparentVersion(conf, TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutVersionManager::maxLayoutVersion)); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 543190140e9f..b9f2d799827d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -696,7 +696,7 @@ private void initializeSystemManagers(OzoneConfiguration conf, } scmLayoutVersionManager = new HDDSLayoutVersionManager( - scmStorageConfig.getLayoutVersion()); + scmStorageConfig.getApparentVersion()); VersionedDatanodeFeatures.initialize(scmLayoutVersionManager); UpgradeFinalizationExecutor @@ -1320,7 +1320,7 @@ public static boolean scmInit(OzoneConfiguration conf, LOG.info("SCM initialization succeeded. Current cluster id for sd={}" + "; cid={}; layoutVersion={}; scmId={}", scmStorageConfig.getStorageDir(), scmStorageConfig.getClusterID(), - scmStorageConfig.getLayoutVersion(), scmStorageConfig.getScmId()); + scmStorageConfig.getApparentVersion(), scmStorageConfig.getScmId()); return true; } catch (IOException ioe) { LOG.error("Could not initialize SCM version file", ioe); @@ -1354,7 +1354,7 @@ public static boolean scmInit(OzoneConfiguration conf, LOG.info("SCM already initialized. Reusing existing cluster id for sd={}" + ";cid={}; layoutVersion={}; HAEnabled={}", scmStorageConfig.getStorageDir(), scmStorageConfig.getClusterID(), - scmStorageConfig.getLayoutVersion(), scmStorageConfig.isSCMHAEnabled()); + scmStorageConfig.getApparentVersion(), scmStorageConfig.isSCMHAEnabled()); return true; } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java index 93f2c7dace88..410e18cb35f3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/SCMUpgradeFinalizer.java @@ -70,7 +70,7 @@ public void finalizeLayoutFeatures(Iterable features, SCMUpgradeF LOG.info("No layout features to finalize."); } } catch (IOException ex) { - throw new UpgradeException(ex, UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED); + throw new UpgradeException(ex, UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index a93fbd4aa99b..06c8e0b32b4a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -753,7 +753,7 @@ public void testProcessLayoutVersionReportHigherMlv() when(scmStorageConfig.getClusterID()).thenReturn("xyz111"); EventPublisher eventPublisher = mock(EventPublisher.class); HDDSLayoutVersionManager lvm = - new HDDSLayoutVersionManager(scmStorageConfig.getLayoutVersion()); + new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion()); SCMContext nodeManagerContext = SCMContext.emptyContext(); SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), @@ -832,7 +832,7 @@ public void testProcessCommandQueueReport() when(scmStorageConfig.getClusterID()).thenReturn("xyz111"); EventPublisher eventPublisher = mock(EventPublisher.class); HDDSLayoutVersionManager lvm = - new HDDSLayoutVersionManager(scmStorageConfig.getLayoutVersion()); + new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion()); createNodeManager(getConf()); SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), @@ -2024,7 +2024,7 @@ public void testNodeOperationalStateChange( SCMStorageConfig scmStorageConfig = mock(SCMStorageConfig.class); when(scmStorageConfig.getClusterID()).thenReturn("xyz111"); EventPublisher eventPublisher = mock(EventPublisher.class); - HDDSLayoutVersionManager lvm = new HDDSLayoutVersionManager(scmStorageConfig.getLayoutVersion()); + HDDSLayoutVersionManager lvm = new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion()); createNodeManager(getConf()); SCMNodeManager nodeManager = new SCMNodeManager(conf, scmStorageConfig, eventPublisher, new NetworkTopologyImpl(conf), diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupInvalidApparentVersion.java similarity index 52% rename from hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java rename to hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupInvalidApparentVersion.java index 25de21766947..cb78e35968be 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupSlvLessThanMlv.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/upgrade/TestScmStartupInvalidApparentVersion.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.upgrade; +import static org.apache.hadoop.hdds.HDDSVersion.SOFTWARE_VERSION; import static org.apache.hadoop.ozone.OzoneConsts.SCM_HA; import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,61 +34,61 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; -import org.apache.hadoop.ozone.upgrade.LayoutFeature; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.ozone.upgrade.UpgradeTestUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; /** - * Tests that SCM will throw an exception on creation when it reads in a - * VERSION file indicating a metadata layout version larger than its - * software layout version. + * Ensures SCM does not start when the VERSION file records an apparent version (still persisted as the layout-version + * integer) that is not supported by the layout version manager for this build + * ({@link HDDSLayoutVersionManager} / {@link HDDSLayoutFeature}). */ -public class TestScmStartupSlvLessThanMlv { +public class TestScmStartupInvalidApparentVersion { + + @TempDir + private Path folder; + + @Test + public void testStartupFailsWhenApparentVersionBetweenLastLayoutFeatureAndZdu() throws Exception { + assertStartupFailsWithComponentVersionMessage( + HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion() + 1); + } @Test - public void testStartupSlvLessThanMlv(@TempDir Path tempDir) + public void testStartupFailsWhenApparentVersionBeyondLastKnownComponentVersion() throws Exception { + assertStartupFailsWithComponentVersionMessage(SOFTWARE_VERSION.serialize() + 1); + } + + private void assertStartupFailsWithComponentVersionMessage(int serializedApparentVersion) throws Exception { - // Add subdirectories under the temporary folder where the version file - // will be placed. - File scmSubdir = tempDir.resolve("scm").resolve("current").toFile(); + File scmSubdir = folder.resolve("scm").resolve("current").toFile(); assertTrue(scmSubdir.mkdirs()); - // Create Ratis directories to simulate a realistic downgrade scenario - // where SCM was previously running with a newer version - File ratisDir = tempDir.resolve("scm.ratis").toFile(); - File snapshotDir = tempDir.resolve("scm.ratis.snapshot").toFile(); + File ratisDir = folder.resolve("scm.ratis").toFile(); + File snapshotDir = folder.resolve("scm.ratis.snapshot").toFile(); assertTrue(ratisDir.mkdirs()); assertTrue(snapshotDir.mkdirs()); OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, - tempDir.toAbsolutePath().toString()); - conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, - tempDir.toAbsolutePath().toString()); + conf.set(ScmConfigKeys.OZONE_SCM_DB_DIRS, folder.toAbsolutePath().toString()); + conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, folder.toAbsolutePath().toString()); - // Set metadata layout version larger then software layout version. - int largestSlv = 0; - for (LayoutFeature f: HDDSLayoutFeature.values()) { - largestSlv = Math.max(largestSlv, f.layoutVersion()); - } - int mlv = largestSlv + 1; + int softwareLayoutVersion = HDDSLayoutVersionManager.maxLayoutVersion(); Properties properties = new Properties(); properties.setProperty(SCM_ID, "scm"); properties.setProperty(SCM_HA, "true"); - // Create version file with MLV > SLV, which should fail the SCM - // construction. - UpgradeTestUtils.createVersionFile(scmSubdir, HddsProtos.NodeType.SCM, mlv, - properties); + UpgradeTestUtils.createVersionFile(scmSubdir, HddsProtos.NodeType.SCM, serializedApparentVersion, properties); + // TODO update this message when SCM migrated to using HDDSVersionManager. + String expectedMessage = String.format( + "Cannot initialize VersionManager. Metadata layout version (%s) > software layout version (%s)", + serializedApparentVersion, softwareLayoutVersion); - Throwable t = assertThrows(IOException.class, - () -> new StorageContainerManager(conf)); - String expectedMessage = String.format("Cannot initialize VersionManager." + - " Metadata layout version (%s) > software layout version (%s)", - mlv, largestSlv); - assertEquals(expectedMessage, t.getMessage()); + IOException ioException = + assertThrows(IOException.class, () -> new StorageContainerManager(conf)); + assertEquals(expectedMessage, ioException.getMessage()); } } diff --git a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java index e447f1011543..b609aa0a00e8 100644 --- a/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java +++ b/hadoop-ozone/cli-repair/src/main/java/org/apache/hadoop/ozone/repair/datanode/schemaupgrade/UpgradeUtils.java @@ -98,7 +98,7 @@ public static Pair getLayoutFeature( DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(conf, dnDetail.getUuidString()); HDDSLayoutVersionManager layoutVersionManager = - new HDDSLayoutVersionManager(layoutStorage.getLayoutVersion()); + new HDDSLayoutVersionManager(layoutStorage.getApparentVersion()); final int metadataLayoutVersion = layoutVersionManager.getMetadataLayoutVersion(); diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java index bee91d466594..91a02f368020 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java @@ -445,7 +445,7 @@ private LayoutVersionProto createLayoutInfo() throws IOException { UUID.randomUUID().toString()); HDDSLayoutVersionManager layoutVersionManager = - new HDDSLayoutVersionManager(layoutStorage.getLayoutVersion()); + new HDDSLayoutVersionManager(layoutStorage.getApparentVersion()); return LayoutVersionProto.newBuilder() .setMetadataLayoutVersion( diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index bdb3cc3cee35..ff2d72c7f6dd 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1599,17 +1599,19 @@ message RangerBGSyncResponse { } message FinalizeUpgradeRequest { + // Ignored by OM; retained for wire compatibility. required string upgradeClientId = 1; } message FinalizeUpgradeResponse { - required hadoop.hdds.UpgradeFinalizationStatus status = 1; + required hadoop.hdds.UpgradeFinalizationStatus status = 1; } message FinalizeUpgradeProgressRequest { + // Ignored by OM; retained for wire compatibility. required string upgradeClientId = 1; - optional bool takeover = 2; - optional bool readonly = 3; + optional bool takeover = 2 [deprecated = true]; + optional bool readonly = 3 [deprecated = true]; } message FinalizeUpgradeProgressResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java index 1c1a4d1f9830..518ae6231144 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java @@ -91,7 +91,7 @@ public class OMStorage extends Storage { */ public OMStorage(OzoneConfiguration conf) throws IOException { super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR, - getInitLayoutVersion(conf, TESTING_INIT_LAYOUT_VERSION_KEY, + getInitApparentVersion(conf, TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutVersionManager::maxLayoutVersion)); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 08634010e44e..b427729df931 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -532,7 +532,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) reconfigurationHandler.setReconfigurationCompleteCallback(reconfigurationHandler.defaultLoggingCallback()); - versionManager = new OMLayoutVersionManager(omStorage.getLayoutVersion()); + versionManager = new OMLayoutVersionManager(omStorage.getApparentVersion()); upgradeFinalizer = new OMUpgradeFinalizer(versionManager); replicationConfigValidator = conf.getObject(ReplicationConfigValidator.class); @@ -1545,13 +1545,13 @@ public static boolean omInit(OzoneConfiguration conf) throws IOException, "OM initialization succeeded.Current cluster id for sd=" + omStorage.getStorageDir() + ";cid=" + omStorage .getClusterID() + ";layoutVersion=" + omStorage - .getLayoutVersion()); + .getApparentVersion()); } else { System.out.println( "OM already initialized.Reusing existing cluster id for sd=" + omStorage.getStorageDir() + ";cid=" + omStorage .getClusterID() + ";layoutVersion=" + omStorage - .getLayoutVersion()); + .getApparentVersion()); } } catch (IOException ioe) { LOG.error("Could not initialize OM version file", ioe); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java index 49bb6fc86587..7bb188af03c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMFinalizeUpgradeResponse.java @@ -36,23 +36,24 @@ public class OMFinalizeUpgradeResponse extends OMClientResponse { private static final Logger LOG = LoggerFactory.getLogger(OMFinalizeUpgradeResponse.class); - private int layoutVersionToWrite = -1; + private int serializedApparentVersion = -1; public OMFinalizeUpgradeResponse( OzoneManagerProtocolProtos.OMResponse omResponse, - int layoutVersionToWrite) { + int serializedApparentVersion) { super(omResponse); - this.layoutVersionToWrite = layoutVersionToWrite; + this.serializedApparentVersion = serializedApparentVersion; } @Override protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - if (layoutVersionToWrite != -1) { - LOG.info("Layout version to persist to DB : {}", layoutVersionToWrite); + if (serializedApparentVersion != -1) { + LOG.info("Serialized apparent component version to persist to DB : {}", + serializedApparentVersion); omMetadataManager.getMetaTable().putWithBatch(batchOperation, LAYOUT_VERSION_KEY, - String.valueOf(layoutVersionToWrite)); + String.valueOf(serializedApparentVersion)); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java new file mode 100644 index 000000000000..26516a172364 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeActionProvider.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider; +import org.reflections.Reflections; +import org.reflections.scanners.SubTypesScanner; +import org.reflections.scanners.TypeAnnotationsScanner; +import org.reflections.util.ConfigurationBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Loads {@link OmUpgradeAction} implementations annotated with {@link UpgradeActionOm} from + * {@link #OM_UPGRADE_CLASS_PACKAGE} only. + */ +public final class OMUpgradeActionProvider implements ComponentUpgradeActionProvider { + + private static final Logger LOG = LoggerFactory.getLogger(OMUpgradeActionProvider.class); + + /** + * Package scanned for {@link UpgradeActionOm}-annotated classes (production OM upgrade actions). + */ + public static final String OM_UPGRADE_CLASS_PACKAGE = "org.apache.hadoop.ozone.om.upgrade"; + + public OMUpgradeActionProvider() { + } + + @Override + public Map load() { + Map upgradeActions = new HashMap<>(); + + Reflections reflections = new Reflections(new ConfigurationBuilder() + .forPackages(OM_UPGRADE_CLASS_PACKAGE) + .setScanners(new TypeAnnotationsScanner(), new SubTypesScanner()) + .setExpandSuperTypes(false) + .setParallel(true)); + Set> typesAnnotatedWith = reflections.getTypesAnnotatedWith(UpgradeActionOm.class); + + typesAnnotatedWith.forEach(actionClass -> { + if (OmUpgradeAction.class.isAssignableFrom(actionClass)) { + try { + OmUpgradeAction action = (OmUpgradeAction) actionClass.getDeclaredConstructor().newInstance(); + UpgradeActionOm annotation = actionClass.getAnnotation(UpgradeActionOm.class); + OMLayoutFeature feature = annotation.feature(); + LOG.info("Registering Upgrade Action : {}", action.name()); + upgradeActions.put(feature, action); + } catch (Exception e) { + LOG.error("Cannot instantiate Upgrade Action class {}", + actionClass.getSimpleName(), e); + } + } else { + LOG.warn("Found upgrade action class not of type " + + "org.apache.hadoop.ozone.om.upgrade.OmUpgradeAction : {}", + actionClass.getName()); + } + }); + + return upgradeActions; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java index f250928b2a97..fcda84149a01 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManager.java @@ -17,32 +17,117 @@ package org.apache.hadoop.ozone.om.upgrade; +import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; + +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.util.Map; import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider; import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.apache.hadoop.ozone.upgrade.UpgradeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Component version manager for Ozone Manager. */ public class OMVersionManager extends ComponentVersionManager { - public OMVersionManager(int serializedApparentVersion) throws IOException { - super(computeApparentVersion(serializedApparentVersion), OzoneManagerVersion.SOFTWARE_VERSION); + + private static final Logger LOG = LoggerFactory.getLogger(OMVersionManager.class); + + private final Map upgradeActions; + + // The OM may not be fully initialized when the version manager is constructed. This field is just provided as an + // argument for upgrade actions when they are run. + private final OzoneManager upgradeActionArg; + + public OMVersionManager(OMStorage storage, OzoneManager upgradeActionArg) throws IOException { + this(storage, upgradeActionArg, new OMUpgradeActionProvider()); + } + + public OMVersionManager(OMStorage storage, OzoneManager upgradeActionArg, + ComponentUpgradeActionProvider upgradeActionProvider) throws IOException { + super(storage, computeApparentVersion(storage.getApparentVersion()), OzoneManagerVersion.SOFTWARE_VERSION); + this.upgradeActionArg = upgradeActionArg; + upgradeActions = upgradeActionProvider.load(); + } + + public void validateDBVersion(OMMetadataManager metadataManager) throws IOException { + ComponentVersion dbVersion = getApparentVersionInDB(metadataManager); + ComponentVersion apparentVersion = getApparentVersion(); + + if (!apparentVersion.equals(dbVersion)) { + LOG.info("Version file has different apparent version ({}) than OM DB ({}). That is expected if this " + + "OM has never been finalized to a newer version.", apparentVersion, dbVersion); + } + } + + public void finalizeFromSnapshotIfRequired(OMMetadataManager metadataManager) throws IOException { + ComponentVersion apparentVersionInDB = getApparentVersionInDB(metadataManager); + if (apparentVersionInDB != null && !isAllowed(apparentVersionInDB)) { + LOG.info("New OM snapshot received with higher apparent version {}. " + + "Attempting to finalize current OM to that version.", apparentVersionInDB); + finalizeUpgrade(); + updateApparentVersionInDB(metadataManager); + } + } + + @VisibleForTesting + public Map getUpgradeActionsForTesting() { + return upgradeActions; + } + + @Override + protected void runUpgradeAction(ComponentVersion componentVersion) throws UpgradeException { + OmUpgradeAction action = upgradeActions.get(componentVersion); + if (action == null) { + return; + } + try { + action.execute(upgradeActionArg); + } catch (Exception e) { + logAndThrow(e, "OM upgrade action for version " + componentVersion + " failed.", + UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED); + } + } + + private static ComponentVersion getApparentVersionInDB(OMMetadataManager metadataManager) throws IOException { + String apparentVersion = metadataManager.getMetaTable().get(LAYOUT_VERSION_KEY); + return (apparentVersion == null) ? null : computeApparentVersion(Integer.parseInt(apparentVersion)); + } + + private void updateApparentVersionInDB(OMMetadataManager metadataManager) throws IOException { + metadataManager.getMetaTable().put(LAYOUT_VERSION_KEY, String.valueOf(getApparentVersion().serialize())); } /** - * If the apparent version stored on the disk is >= 100, it indicates the component has been finalized for the - * ZDU feature, and the apparent version corresponds to a version in {@link OzoneManagerVersion}. - * If the apparent version stored on the disk is < 100, it indicates the component is not yet finalized for the - * ZDU feature, and the apparent version corresponds to a version in {@link OMLayoutFeature}. + * Maps a serialized apparent version to a {@link ComponentVersion}. + * If the value is >= {@link OzoneManagerVersion#ZDU} serialized, the OM has been finalized for ZDU and the + * apparent version is resolved via {@link OzoneManagerVersion#deserialize(int)}. Values with no matching + * {@link OzoneManagerVersion} fail startup with the persisted integer in the exception message. + * If the value is below that threshold, the apparent version is resolved as an {@link OMLayoutFeature}. Integers in + * the gap between the largest {@link OMLayoutFeature} and ZDU are not valid legacy layout values; startup fails with + * the persisted integer in the exception message. */ - private static ComponentVersion computeApparentVersion(int serializedApparentVersion) { - if (serializedApparentVersion < OzoneManagerVersion.ZDU.serialize()) { - return OMLayoutFeature.deserialize(serializedApparentVersion); + private static ComponentVersion computeApparentVersion(int serializedApparentVersion) throws IOException { + if (serializedApparentVersion >= OzoneManagerVersion.ZDU.serialize()) { + OzoneManagerVersion fromOm = OzoneManagerVersion.deserialize(serializedApparentVersion); + if (fromOm != OzoneManagerVersion.FUTURE_VERSION) { + return fromOm; + } } else { - return OzoneManagerVersion.deserialize(serializedApparentVersion); + ComponentVersion fromLayout = OMLayoutFeature.deserialize(serializedApparentVersion); + if (fromLayout != null) { + return fromLayout; + } } + throw new IOException("Initialization failed. Disk contains unknown apparent version " + serializedApparentVersion + + " for software version " + OzoneManagerVersion.SOFTWARE_VERSION + ". Make sure OM was not downgraded after" + + " finalization"); } - - // TODO HDDS-14826: Register upgrade actions based on annotations } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index bea7785bfbc2..87d3d83002cb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -507,7 +507,7 @@ public void validateRequest(OMRequest omRequest) throws OMException { OMException.ResultCodes.INVALID_REQUEST); } - // Layout version should have been set up the leader while serializing + // Apparent version should have been set up the leader while serializing // the request, and hence cannot be null. This version is used by each // node to identify which request handler version to use. if (omRequest.getLayoutVersion() == null) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java index 12b88c11e673..ce85249c1a7a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMStorage.java @@ -282,7 +282,7 @@ private void setupAPersistedVersionFileWithNodeId( OzoneConfiguration conf, String nodeId) throws IOException { OMStorage storage = new OMStorage(conf); storage.setClusterId("clusterId"); - storage.setLayoutVersion(OMLayoutVersionManager.maxLayoutVersion()); + storage.setApparentVersion(OMLayoutVersionManager.maxLayoutVersion()); storage.setOmId(OM_ID_STR); if (nodeId != null) { storage.setOmNodeId(nodeId); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManagerTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManagerTestUtils.java new file mode 100644 index 000000000000..d2549431274e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/OMVersionManagerTestUtils.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.ozone.OzoneManagerVersion; + +/** + * Mockito helpers for {@link OMVersionManager} in unit tests. + */ +public final class OMVersionManagerTestUtils { + + private OMVersionManagerTestUtils() { + } + + /** + * Mock with apparent and software version both {@link OzoneManagerVersion#SOFTWARE_VERSION}, + * {@link OMVersionManager#needsFinalization()} false, and {@link OMVersionManager#isAllowed(ComponentVersion)} + * true for any argument. + */ + public static OMVersionManager mockFinalizedOmVersionManager() { + OMVersionManager ovm = mock(OMVersionManager.class); + when(ovm.getApparentVersion()).thenReturn(OzoneManagerVersion.SOFTWARE_VERSION); + when(ovm.getSoftwareVersion()).thenReturn(OzoneManagerVersion.SOFTWARE_VERSION); + when(ovm.needsFinalization()).thenReturn(false); + when(ovm.isAllowed(any(ComponentVersion.class))).thenReturn(true); + return ovm; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizer.java index 27d6c8f3a7a5..f3318758d42b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizer.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om.upgrade; -import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.LAYOUT_FEATURE_FINALIZATION_FAILED; +import static org.apache.hadoop.ozone.upgrade.UpgradeException.ResultCodes.UPGRADE_FINALIZATION_FAILED; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.ALREADY_FINALIZED; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.FINALIZATION_DONE; import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.Status.FINALIZATION_REQUIRED; @@ -163,13 +163,13 @@ public void testFinalizationWithUpgradeAction() throws Exception { // the first feature has an upgrade action, and the action execution is // checked by verifying on om.getVersion verify(om.getOmStorage(), once()) - .setLayoutVersion(f.layoutVersion()); + .setApparentVersion(f.layoutVersion()); verify(om, once()).getVersion(); // The second feature has a NOOP, but should update the layout version. f = it.next(); verify(om.getOmStorage(), once()) - .setLayoutVersion(f.layoutVersion()); + .setApparentVersion(f.layoutVersion()); if (finalizer.isFinalizationDone()) { when(versionManager.getUpgradeState()).thenReturn(FINALIZATION_DONE); @@ -195,7 +195,7 @@ public void testFinalizationWithFailingUpgradeAction() throws Exception { OMUpgradeFinalizer finalizer = new OMUpgradeFinalizer(versionManager); UpgradeException e = assertThrows(UpgradeException.class, () -> finalizer.finalize(CLIENT_ID, om)); assertThat(e.getMessage()).contains(lfs.iterator().next().toString()); - assertEquals(e.getResult(), LAYOUT_FEATURE_FINALIZATION_FAILED); + assertEquals(e.getResult(), UPGRADE_FINALIZATION_FAILED); if (finalizer.isFinalizationDone()) { when(versionManager.getUpgradeState()).thenReturn(FINALIZATION_DONE); } @@ -204,12 +204,12 @@ public void testFinalizationWithFailingUpgradeAction() throws Exception { Iterator it = lfs.iterator(); OMLayoutFeature f = it.next(); verify(om.getOmStorage(), never()) - .setLayoutVersion(f.layoutVersion()); + .setApparentVersion(f.layoutVersion()); // Verify that we never got to the second feature. f = it.next(); verify(om.getOmStorage(), never()) - .setLayoutVersion(f.layoutVersion()); + .setApparentVersion(f.layoutVersion()); StatusAndMessages status = finalizer.reportStatus(CLIENT_ID, false); assertEquals(FINALIZATION_DONE, status.status()); @@ -262,9 +262,9 @@ private OzoneManager mockOzoneManager(int initialLayoutVersion) { (Answer) inv -> { storedLayoutVersion = inv.getArgument(0, Integer.class); return null; - }).when(st).setLayoutVersion(anyInt()); + }).when(st).setApparentVersion(anyInt()); - lenient().when(st.getLayoutVersion()) + lenient().when(st.getApparentVersion()) .thenAnswer((Answer) ignore -> storedLayoutVersion); when(mock.getOmStorage()).thenReturn(st); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java index eed52733acbb..9e89a7780dee 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java @@ -17,34 +17,83 @@ package org.apache.hadoop.ozone.om.upgrade; +import static org.apache.hadoop.ozone.OzoneManagerVersion.SOFTWARE_VERSION; +import static org.apache.hadoop.ozone.OzoneManagerVersion.ZDU; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.DELEGATION_TOKEN_SYMMETRIC_SIGN; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.HBASE_SUPPORT; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.QUOTA; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import org.apache.hadoop.hdds.ComponentVersion; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.upgrade.AbstractComponentVersionManagerTest; -import org.apache.hadoop.ozone.upgrade.ComponentVersionManager; +import org.apache.hadoop.ozone.upgrade.ComponentUpgradeActionProvider; +import org.apache.hadoop.ozone.upgrade.UpgradeException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.provider.Arguments; +import org.mockito.Mockito; /** - * Tests for {@link OMVersionManager}. + * Tests for {@link OMVersionManager}. Shared abstract tests use an empty upgrade-action map; action behavior is + * covered with map-based providers and a required classpath discovery test. */ class TestOMVersionManager extends AbstractComponentVersionManagerTest { private static final List ALL_VERSIONS; + private OzoneConfiguration conf; + + @TempDir + private Path tempFolder; + static { ALL_VERSIONS = new ArrayList<>(Arrays.asList(OMLayoutFeature.values())); for (OzoneManagerVersion version : OzoneManagerVersion.values()) { // Add all defined versions after and including ZDU to get the complete version list. - if (OzoneManagerVersion.ZDU.isSupportedBy(version) && version != OzoneManagerVersion.FUTURE_VERSION) { + if (ZDU.isSupportedBy(version) && version != OzoneManagerVersion.FUTURE_VERSION) { ALL_VERSIONS.add(version); } } } + @BeforeEach + public void init() { + conf = new OzoneConfiguration(); + } + public static Stream preFinalizedVersionArgs() { return ALL_VERSIONS.stream() .limit(ALL_VERSIONS.size() - 1) @@ -52,8 +101,17 @@ public static Stream preFinalizedVersionArgs() { } @Override - protected ComponentVersionManager createManager(int serializedApparentVersion) throws IOException { - return new OMVersionManager(serializedApparentVersion); + protected OMVersionManager createManager(int serializedApparentVersion) throws IOException { + // By default create a version manager which does not have any upgrade actions to run. Production upgrade actions + // may not be able to run in a test environment during finalization. + return createManager(serializedApparentVersion, HashMap::new); + } + + private OMVersionManager createManager(int serializedApparentVersion, + ComponentUpgradeActionProvider actions) throws IOException { + OMStorage storage = newOmStorage(serializedApparentVersion); + OzoneManager mockOM = Mockito.mock(OzoneManager.class); + return new OMVersionManager(storage, mockOM, actions); } @Override @@ -65,4 +123,104 @@ protected List allVersionsInOrder() { protected ComponentVersion expectedSoftwareVersion() { return OzoneManagerVersion.SOFTWARE_VERSION; } + + @Test + public void testClasspathScanDiscoversUpgradeActions() throws Exception { + // Regardless of whether OM is finalized, the same set of upgrade actions should be loaded. + try (OMVersionManager versionManager = createManager(INITIAL_VERSION.serialize(), new OMUpgradeActionProvider())) { + assertTrue(versionManager.needsFinalization()); + OmUpgradeAction quotaAction = versionManager.getUpgradeActionsForTesting().get(QUOTA); + assertInstanceOf(QuotaRepairUpgradeAction.class, quotaAction); + } + + try (OMVersionManager versionManager = createManager(SOFTWARE_VERSION.serialize(), new OMUpgradeActionProvider())) { + assertFalse(versionManager.needsFinalization()); + OmUpgradeAction quotaAction = versionManager.getUpgradeActionsForTesting().get(QUOTA); + assertInstanceOf(QuotaRepairUpgradeAction.class, quotaAction); + } + } + + @Test + public void testFinalizeRunsSuppliedUpgradeAction() throws Exception { + OmUpgradeAction mockECAction = mock(OmUpgradeAction.class); + OmUpgradeAction mockZDUAction = mock(OmUpgradeAction.class); + + ComponentUpgradeActionProvider provider = () -> { + Map m = new HashMap<>(); + m.put(ERASURE_CODED_STORAGE_SUPPORT, mockECAction); + m.put(ZDU, mockZDUAction); + return m; + }; + + try (OMVersionManager versionManager = createManager(QUOTA.serialize(), provider)) { + versionManager.finalizeUpgrade(); + assertEquals(OzoneManagerVersion.SOFTWARE_VERSION, versionManager.getApparentVersion()); + + // QUOTA was added after EC, so the EC upgrade action should not run when we finalize from this version. + verify(mockECAction, never()).execute(any()); + verify(mockZDUAction, atLeastOnce()).execute(any()); + assertOmApparentVersionOnDisk(conf, OzoneManagerVersion.SOFTWARE_VERSION.serialize()); + } + } + + @Test + public void testUpgradeActionFailureAbortsFinalize() throws Exception { + ComponentUpgradeActionProvider provider = () -> { + Map m = new HashMap<>(); + m.put(DELEGATION_TOKEN_SYMMETRIC_SIGN, o -> { + throw new IOException("expected test failure"); + }); + return m; + }; + + try (OMVersionManager versionManager = createManager(QUOTA.serialize(), provider)) { + UpgradeException thrown = + assertThrows(UpgradeException.class, versionManager::finalizeUpgrade); + assertEquals(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED, thrown.getResult()); + // HBase is the version before symmetric encrypted delegation tokens, which has failed. + assertEquals(HBASE_SUPPORT, versionManager.getApparentVersion()); + assertOmApparentVersionOnDisk(conf, HBASE_SUPPORT.serialize()); + } + } + + @Test + public void testPersistFailureRollsBack() throws Exception { + // Create a mock storage instance that throws when persisting version updates. + OMStorage storage = mock(OMStorage.class); + AtomicInteger persistedApparentVersion = new AtomicInteger(INITIAL_VERSION.serialize()); + when(storage.getApparentVersion()).thenAnswer(invocation -> persistedApparentVersion.get()); + doAnswer(invocation -> { + persistedApparentVersion.set(invocation.getArgument(0)); + return null; + }).when(storage).setApparentVersion(anyInt()); + doThrow(new IOException("persist failed")).when(storage).persistCurrentState(); + + OzoneManager mockOM = Mockito.mock(OzoneManager.class); + try (OMVersionManager versionManager = new OMVersionManager(storage, mockOM, HashMap::new)) { + assertEquals(INITIAL_VERSION, versionManager.getApparentVersion()); + UpgradeException thrown = assertThrows(UpgradeException.class, versionManager::finalizeUpgrade); + assertEquals(UpgradeException.ResultCodes.APPARENT_VERSION_UPDATE_FAILED, thrown.getResult()); + assertEquals(INITIAL_VERSION, versionManager.getApparentVersion()); + assertEquals(INITIAL_VERSION.serialize(), storage.getApparentVersion()); + } + } + + private OMStorage newOmStorage(int apparentVersion) + throws IOException { + // Reinitialize the configuration to point to a new unique storage location. + Path dbDir = Files.createDirectory(new File(tempFolder.toFile(), UUID.randomUUID().toString()).toPath()); + conf.set(OMConfigKeys.OZONE_OM_DB_DIRS, dbDir.toString()); + OMStorage storage = new OMStorage(conf); + storage.setClusterId("test-cluster"); + storage.setApparentVersion(apparentVersion); + storage.setOmId(UUID.randomUUID().toString()); + storage.initialize(); + storage.persistCurrentState(); + return storage; + } + + private static void assertOmApparentVersionOnDisk(OzoneConfiguration conf, int expected) throws IOException { + OMStorage reloaded = new OMStorage(conf); + assertEquals(expected, reloaded.getApparentVersion()); + } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java index 57067c421344..2db9fc153b57 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java @@ -218,7 +218,7 @@ public ReconStorageContainerManagerFacade(OzoneConfiguration conf, this.dbStore = DBStoreBuilder.createDBStore(ozoneConfiguration, ReconSCMDBDefinition.get()); HDDSLayoutVersionManager scmLayoutVersionManager = - new HDDSLayoutVersionManager(scmStorageConfig.getLayoutVersion()); + new HDDSLayoutVersionManager(scmStorageConfig.getApparentVersion()); this.scmhaManager = SCMHAManagerStub.getInstance( true, new SCMDBTransactionBufferImpl()); this.sequenceIdGen = new SequenceIdGenerator( diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java index e0eb2f020ccb..8aa49a991741 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java @@ -82,7 +82,7 @@ public void setUp() throws Exception { ReconUtils reconUtils = new ReconUtils(); ReconStorageConfig reconStorageConfig = new ReconStorageConfig(conf, reconUtils); versionManager = new HDDSLayoutVersionManager( - reconStorageConfig.getLayoutVersion()); + reconStorageConfig.getApparentVersion()); store = DBStoreBuilder.createDBStore(conf, ReconSCMDBDefinition.get()); reconContext = new ReconContext(conf, reconUtils); }