HDDS-15848. Add new upgrade action annotations for component versions

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/upgrade/DatanodeUpgradeActionProvider.java

@@ -17,24 +17,32 @@
 
 package org.apache.hadoop.hdds.upgrade;
 
+import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.ozone.upgrade.AbstractUpgradeActionProvider;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
+import org.apache.hadoop.ozone.upgrade.DatanodeUpgradeActionForLayoutFeature;
+import org.apache.hadoop.ozone.upgrade.DatanodeUpgradeActionForVersion;
 
 /**
- * Loads {@link DatanodeUpgradeAction} implementations annotated with {@link UpgradeActionDatanode}.
+ * Loads {@link DatanodeUpgradeAction} implementations annotated with {@link DatanodeUpgradeActionForLayoutFeature} or
+ * {@link DatanodeUpgradeActionForVersion}.
  */
 public final class DatanodeUpgradeActionProvider extends AbstractUpgradeActionProvider<DatanodeUpgradeAction> {
 
   public static final String DATANODE_UPGRADE_CLASS_PACKAGE = "org.apache.hadoop.ozone.container";
 
   public DatanodeUpgradeActionProvider() {
-    super(UpgradeActionDatanode.class, DatanodeUpgradeAction.class, DATANODE_UPGRADE_CLASS_PACKAGE);
+    super(ImmutableSet.of(DatanodeUpgradeActionForLayoutFeature.class, DatanodeUpgradeActionForVersion.class),
+        DatanodeUpgradeAction.class, DATANODE_UPGRADE_CLASS_PACKAGE);
   }
 
   @Override
   protected ComponentVersion extractVersion(Class<?> clazz) {
-    UpgradeActionDatanode annotation = clazz.getAnnotation(UpgradeActionDatanode.class);
-    return annotation.feature();
+    DatanodeUpgradeActionForLayoutFeature layoutAnnotation =
+        clazz.getAnnotation(DatanodeUpgradeActionForLayoutFeature.class);
+    if (layoutAnnotation != null) {
+      return layoutAnnotation.feature();
+    }
+    return clazz.getAnnotation(DatanodeUpgradeActionForVersion.class).version();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ContainerTableSchemaFinalizeAction.java

@@ -31,14 +31,14 @@
 import org.apache.hadoop.ozone.container.metadata.WitnessedContainerDBDefinition;
 import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStore;
 import org.apache.hadoop.ozone.container.metadata.WitnessedContainerMetadataStoreImpl;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
+import org.apache.hadoop.ozone.upgrade.DatanodeUpgradeActionForLayoutFeature;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Upgrade Action for DataNode for update the table schema data of containerIds Table.
  */
-@UpgradeActionDatanode(feature = WITNESSED_CONTAINER_DB_PROTO_VALUE)
+@DatanodeUpgradeActionForLayoutFeature(feature = WITNESSED_CONTAINER_DB_PROTO_VALUE)
 public class ContainerTableSchemaFinalizeAction
     implements DatanodeUpgradeAction {
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV2FinalizeAction.java

@@ -21,14 +21,14 @@
 
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
+import org.apache.hadoop.ozone.upgrade.DatanodeUpgradeActionForLayoutFeature;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Upgrade Action for DataNode for the very first first Upgrade Version.
  */
-@UpgradeActionDatanode(feature = DATANODE_SCHEMA_V2)
+@DatanodeUpgradeActionForLayoutFeature(feature = DATANODE_SCHEMA_V2)
 public class DatanodeSchemaV2FinalizeAction
     implements DatanodeUpgradeAction {
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeSchemaV3FinalizeAction.java

@@ -26,15 +26,15 @@
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
 import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
+import org.apache.hadoop.ozone.upgrade.DatanodeUpgradeActionForLayoutFeature;
 import org.apache.ratis.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Upgrade Action for DataNode for SCHEMA V3.
  */
-@UpgradeActionDatanode(feature = DATANODE_SCHEMA_V3)
+@DatanodeUpgradeActionForLayoutFeature(feature = DATANODE_SCHEMA_V3)
 public class DatanodeSchemaV3FinalizeAction
     implements DatanodeUpgradeAction {
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java

@@ -29,14 +29,14 @@
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
 import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
+import org.apache.hadoop.ozone.upgrade.DatanodeUpgradeActionForLayoutFeature;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Action to run upgrade flow for SCM HA exactly once.
  */
-@UpgradeActionDatanode(feature = SCM_HA)
+@DatanodeUpgradeActionForLayoutFeature(feature = SCM_HA)
 public class ScmHAFinalizeUpgradeActionDatanode
     implements DatanodeUpgradeAction {
   private static final Logger LOG =

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionScm.java → hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/DatanodeUpgradeActionForLayoutFeature.java

@@ -24,10 +24,10 @@
 import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
 
 /**
- * Annotation to specify an SCM upgrade action.
+ * Annotation to specify a Datanode upgrade action tied to an HDDS layout feature.
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
-public @interface UpgradeActionScm {
+public @interface DatanodeUpgradeActionForLayoutFeature {
   HDDSLayoutFeature feature();
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/DatanodeUpgradeActionForVersion.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.hadoop.hdds.HDDSVersion;
+
+/**
+ * Annotation to specify a Datanode upgrade action tied to an HDDS component version.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface DatanodeUpgradeActionForVersion {
+  HDDSVersion version();
+}

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeVersionManager.java

@@ -141,13 +141,17 @@ public void testClasspathScanDiscoversUpgradeActions() throws Exception {
       assertTrue(versionManager.needsFinalization());
       DatanodeUpgradeAction quotaAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V3);
       assertInstanceOf(DatanodeSchemaV3FinalizeAction.class, quotaAction);
+      DatanodeUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+      assertInstanceOf(ZduDatanodeUpgradeActionForTest.class, zduAction);
     }
 
     try (DatanodeVersionManager versionManager =
         createManager(HDDSVersion.SOFTWARE_VERSION.serialize(), new DatanodeUpgradeActionProvider())) {
       assertFalse(versionManager.needsFinalization());
       DatanodeUpgradeAction quotaAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V3);
       assertInstanceOf(DatanodeSchemaV3FinalizeAction.class, quotaAction);
+      DatanodeUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+      assertInstanceOf(ZduDatanodeUpgradeActionForTest.class, zduAction);
     }
   }
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeUpgradeActionProvider.java → hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/ZduDatanodeUpgradeActionForTest.java

@@ -17,25 +17,19 @@
 
 package org.apache.hadoop.ozone.container.upgrade;
 
-import org.apache.hadoop.hdds.ComponentVersion;
+import static org.apache.hadoop.hdds.HDDSVersion.ZDU;
+
 import org.apache.hadoop.hdds.upgrade.DatanodeUpgradeAction;
-import org.apache.hadoop.ozone.upgrade.AbstractUpgradeActionProvider;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionDatanode;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.upgrade.DatanodeUpgradeActionForVersion;
 
 /**
- * Loads {@link DatanodeUpgradeAction} implementations annotated with {@link UpgradeActionDatanode}.
+ * No-op upgrade action used only to verify that {@link DatanodeUpgradeActionForVersion} is scanned by
+ * {@link org.apache.hadoop.hdds.upgrade.DatanodeUpgradeActionProvider} in tests.
  */
-public final class DatanodeUpgradeActionProvider extends AbstractUpgradeActionProvider<DatanodeUpgradeAction> {
-
-  public static final String DATANODE_UPGRADE_CLASS_PACKAGE = "org.apache.hadoop.ozone.container";
-
-  public DatanodeUpgradeActionProvider() {
-    super(UpgradeActionDatanode.class, DatanodeUpgradeAction.class, DATANODE_UPGRADE_CLASS_PACKAGE);
-  }
-
+@DatanodeUpgradeActionForVersion(version = ZDU)
+public class ZduDatanodeUpgradeActionForTest implements DatanodeUpgradeAction {
   @Override
-  protected ComponentVersion extractVersion(Class<?> clazz) {
-    UpgradeActionDatanode annotation = clazz.getAnnotation(UpgradeActionDatanode.class);
-    return annotation.feature();
+  public void execute(DatanodeStateMachine arg) {
   }
 }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractUpgradeActionProvider.java

@@ -19,6 +19,7 @@
 
 import java.lang.annotation.Annotation;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import org.apache.hadoop.hdds.ComponentVersion;
@@ -39,14 +40,14 @@ public abstract class AbstractUpgradeActionProvider<T extends UpgradeAction<?>>
 
   private static final Logger LOG = LoggerFactory.getLogger(AbstractUpgradeActionProvider.class);
 
-  private final Class<? extends Annotation> annotationClass;
+  private final Set<Class<? extends Annotation>> annotationClasses;
   private final Class<T> actionClass;
   private final String[] packagesToScan;
 
-  protected AbstractUpgradeActionProvider(Class<? extends Annotation> annotationClass,
+  protected AbstractUpgradeActionProvider(Set<Class<? extends Annotation>> annotationClasses,
                                           Class<T> actionClass,
                                           String... packagesToScan) {
-    this.annotationClass = annotationClass;
+    this.annotationClasses = annotationClasses;
     this.actionClass = actionClass;
     this.packagesToScan = packagesToScan;
   }
@@ -60,16 +61,24 @@ public Map<ComponentVersion, T> load() {
         .setScanners(new TypeAnnotationsScanner(), new SubTypesScanner())
         .setExpandSuperTypes(false)
         .setParallel(true));
-    Set<Class<?>> typesAnnotatedWith = reflections.getTypesAnnotatedWith(annotationClass);
+    Set<Class<?>> typesAnnotatedWith = new HashSet<>();
+    for (Class<? extends Annotation> annotationClass : annotationClasses) {
+      typesAnnotatedWith.addAll(reflections.getTypesAnnotatedWith(annotationClass));
+    }
 
     typesAnnotatedWith.forEach(clazz -> {
+      long annotationCount = annotationClasses.stream().filter(clazz::isAnnotationPresent).count();
+      if (annotationCount > 1) {
+        throw new IllegalStateException("Upgrade action class " + clazz.getName()
+            + " has more than one upgrade action annotation. Only one is allowed.");
+      }
       if (actionClass.isAssignableFrom(clazz)) {
         try {
           @SuppressWarnings("unchecked")
           T action = (T) clazz.getDeclaredConstructor().newInstance();
-          ComponentVersion feature = extractVersion(clazz);
+          ComponentVersion version = extractVersion(clazz);
           LOG.info("Registering Upgrade Action : {}", action.name());
-          upgradeActions.put(feature, action);
+          upgradeActions.put(version, action);
         } catch (Exception e) {
           LOG.error("Cannot instantiate Upgrade Action class {}",
               clazz.getSimpleName(), e);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/upgrade/ScmOnFinalizeActionForDatanodeSchemaV2.java

@@ -21,14 +21,14 @@
 
 import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
 import org.apache.hadoop.hdds.upgrade.ScmUpgradeAction;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionScm;
+import org.apache.hadoop.ozone.upgrade.ScmUpgradeActionForLayoutFeature;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * SCM Upgrade Action for the very first Upgrade Version.
  */
-@UpgradeActionScm(feature = DATANODE_SCHEMA_V2)
+@ScmUpgradeActionForLayoutFeature(feature = DATANODE_SCHEMA_V2)
 public class ScmOnFinalizeActionForDatanodeSchemaV2 implements ScmUpgradeAction {
   private static final Logger LOG = LoggerFactory.getLogger(ScmOnFinalizeActionForDatanodeSchemaV2.class);
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/upgrade/ScmUpgradeActionProvider.java

@@ -17,24 +17,31 @@
 
 package org.apache.hadoop.hdds.upgrade;
 
+import com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.hdds.ComponentVersion;
 import org.apache.hadoop.ozone.upgrade.AbstractUpgradeActionProvider;
-import org.apache.hadoop.ozone.upgrade.UpgradeActionScm;
+import org.apache.hadoop.ozone.upgrade.ScmUpgradeActionForLayoutFeature;
+import org.apache.hadoop.ozone.upgrade.ScmUpgradeActionForVersion;
 
 /**
- * Loads {@link ScmUpgradeAction} implementations annotated with {@link UpgradeActionScm}.
+ * Loads {@link ScmUpgradeAction} implementations annotated with {@link ScmUpgradeActionForLayoutFeature} or
+ * {@link ScmUpgradeActionForVersion}.
  */
 public final class ScmUpgradeActionProvider extends AbstractUpgradeActionProvider<ScmUpgradeAction> {
 
   public static final String SCM_UPGRADE_CLASS_PACKAGE = "org.apache.hadoop.hdds.scm.server";
 
   public ScmUpgradeActionProvider() {
-    super(UpgradeActionScm.class, ScmUpgradeAction.class, SCM_UPGRADE_CLASS_PACKAGE);
+    super(ImmutableSet.of(ScmUpgradeActionForLayoutFeature.class, ScmUpgradeActionForVersion.class),
+        ScmUpgradeAction.class, SCM_UPGRADE_CLASS_PACKAGE);
   }
 
   @Override
   protected ComponentVersion extractVersion(Class<?> clazz) {
-    UpgradeActionScm annotation = clazz.getAnnotation(UpgradeActionScm.class);
-    return annotation.feature();
+    ScmUpgradeActionForLayoutFeature layoutAnnotation = clazz.getAnnotation(ScmUpgradeActionForLayoutFeature.class);
+    if (layoutAnnotation != null) {
+      return layoutAnnotation.feature();
+    }
+    return clazz.getAnnotation(ScmUpgradeActionForVersion.class).version();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/upgrade/UpgradeActionDatanode.java → hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/ScmUpgradeActionForLayoutFeature.java

@@ -24,10 +24,10 @@
 import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
 
 /**
- * Annotation to specify a Datanode upgrade action.
+ * Annotation to specify an SCM upgrade action tied to an HDDS layout feature.
  */
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
-public @interface UpgradeActionDatanode {
+public @interface ScmUpgradeActionForLayoutFeature {
   HDDSLayoutFeature feature();
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/upgrade/ScmUpgradeActionForVersion.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.upgrade;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.apache.hadoop.hdds.HDDSVersion;
+
+/**
+ * Annotation to specify an SCM upgrade action tied to an HDDS component version.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface ScmUpgradeActionForVersion {
+  HDDSVersion version();
+}

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/upgrade/TestScmVersionManager.java

@@ -139,13 +139,17 @@ public void testClasspathScanDiscoversUpgradeActions() throws Exception {
       assertTrue(versionManager.needsFinalization());
       ScmUpgradeAction upgradeAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V2);
       assertInstanceOf(ScmOnFinalizeActionForDatanodeSchemaV2.class, upgradeAction);
+      ScmUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+      assertInstanceOf(ZduScmUpgradeActionForTest.class, zduAction);
     }
 
     try (ScmVersionManager versionManager = createManager(HDDSVersion.SOFTWARE_VERSION.serialize(),
         new ScmUpgradeActionProvider())) {
       assertFalse(versionManager.needsFinalization());
       ScmUpgradeAction upgradeAction = versionManager.getUpgradeActionsForTesting().get(DATANODE_SCHEMA_V2);
       assertInstanceOf(ScmOnFinalizeActionForDatanodeSchemaV2.class, upgradeAction);
+      ScmUpgradeAction zduAction = versionManager.getUpgradeActionsForTesting().get(HDDSVersion.ZDU);
+      assertInstanceOf(ZduScmUpgradeActionForTest.class, zduAction);
     }
   }