HDDS-15138. Add EC DN safemode rule and control RATIS/THREE background pipelines for EC-default clusters.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -459,6 +459,18 @@ public final class ScmConfigKeys {
   public static final boolean
       OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE_DEFAULT = true;
 
+  /**
+   * If true, BackgroundPipelineCreator will create RATIS/THREE pipelines even
+   * when the default replication is EC. This keeps RATIS write paths warm for
+   * mixed-workload clusters. If false, RATIS/THREE pipeline creation is
+   * skipped for EC-default clusters.
+   */
+  public static final String OZONE_SCM_PIPELINE_CREATE_RATIS_THREE =
+      "ozone.scm.pipeline.creation.ratis.three";
+
+  public static final boolean
+      OZONE_SCM_PIPELINE_CREATE_RATIS_THREE_DEFAULT = true;
+
   public static final String OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR =
       "ozone.scm.block.deletion.per.dn.distribution.factor";
 

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1709,6 +1709,17 @@
       If enabled, SCM will auto create RATIS factor ONE pipeline.
     </description>
   </property>
+  <property>
+    <name>ozone.scm.pipeline.creation.ratis.three</name>
+    <value>true</value>
+    <tag>OZONE, SCM, PIPELINE</tag>
+    <description>
+      When true, SCM creates RATIS/THREE pipelines in the background and
+      requires them during safemode. Applies only when the cluster default
+      replication type is EC. For RATIS-default clusters this flag has no
+      effect.
+    </description>
+  </property>
   <property>
     <name>hdds.scm.safemode.threshold.pct</name>
     <value>0.99</value>

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java

@@ -45,6 +45,7 @@
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.concurrent.BlockingQueue;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -54,6 +55,7 @@
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReport;
 import org.apache.hadoop.hdds.security.SecurityConfig;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.ha.ConfUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
@@ -218,4 +220,25 @@ public static void checkIfCertSignRequestAllowed(
       }
     }
   }
+
+  /**
+   * Returns default replication config, or null when configured values are
+   * invalid. Callers can decide whether to fallback or skip their operation.
+   */
+  public static ReplicationConfig getDefaultReplicationConfig(
+      ConfigurationSource conf, Logger logger, String componentName) {
+    try {
+      return ReplicationConfig.getDefault(conf);
+    } catch (IllegalArgumentException e) {
+      logger.warn("Ignoring invalid default replication config in {}: "
+              + "type={}, replication={}.",
+          componentName,
+          conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+              OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT),
+          conf.get(OzoneConfigKeys.OZONE_REPLICATION,
+              OzoneConfigKeys.OZONE_REPLICATION_DEFAULT),
+          e);
+      return null;
+    }
+  }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java

@@ -25,6 +25,7 @@
 import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.PRE_CHECK_COMPLETED;
 import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.io.IOException;
 import java.time.Clock;
@@ -43,9 +44,9 @@
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmUtils;
 import org.apache.hadoop.hdds.scm.ha.SCMContext;
 import org.apache.hadoop.hdds.scm.ha.SCMService;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -88,6 +89,7 @@ public class BackgroundPipelineCreator implements SCMService {
   private final AtomicBoolean running = new AtomicBoolean(false);
   private final long intervalInMillis;
   private final Clock clock;
+  private final boolean createRatisThreeForEcDefault;
 
   BackgroundPipelineCreator(PipelineManager pipelineManager,
       ConfigurationSource conf, SCMContext scmContext, Clock clock) {
@@ -109,6 +111,9 @@ public class BackgroundPipelineCreator implements SCMService {
         ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL,
         ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT,
         TimeUnit.MILLISECONDS);
+    this.createRatisThreeForEcDefault = conf.getBoolean(
+        ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE,
+        ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE_DEFAULT);
 
     threadName = scmContext.threadNamePrefix() + THREAD_NAME;
   }
@@ -204,42 +209,18 @@ private boolean skipCreation(ReplicationConfig replicationConfig,
   }
 
   private void createPipelines() throws RuntimeException {
-    // TODO: #CLUTIL Different replication factor may need to be supported
-    HddsProtos.ReplicationType type = HddsProtos.ReplicationType.valueOf(
-        conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
-            OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
-    boolean autoCreateFactorOne = conf.getBoolean(
-        ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE,
+    boolean autoCreateFactorOne = conf.getBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE,
         ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE_DEFAULT);
 
-    List<ReplicationConfig> list =
-        new ArrayList<>();
-    for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor
-        .values()) {
-      if (factor == ReplicationFactor.ZERO) {
-        continue; // Ignore it.
-      }
-      final ReplicationConfig replicationConfig;
-      if (type != EC) {
-        replicationConfig =
-            ReplicationConfig.fromProtoTypeAndFactor(type, factor);
-      } else if (factor == ReplicationFactor.ONE) {
-        replicationConfig =
-            ReplicationConfig.fromProtoTypeAndFactor(RATIS, factor);
-      } else {
-        continue;
-      }
-      if (skipCreation(replicationConfig, autoCreateFactorOne)) {
-        // Skip this iteration for creating pipeline
-        continue;
-      }
-      list.add(replicationConfig);
+    List<ReplicationConfig> list = getReplicationConfigs(autoCreateFactorOne);
+    if (list.isEmpty()) {
+      LOG.debug("No replication configs selected for background pipeline creation.");
+      return;
     }
 
     LoopingIterator it = new LoopingIterator(list);
     while (it.hasNext()) {
-      ReplicationConfig replicationConfig =
-          (ReplicationConfig) it.next();
+      ReplicationConfig replicationConfig = (ReplicationConfig) it.next();
 
       try {
         Pipeline pipeline = pipelineManager.createPipeline(replicationConfig);
@@ -255,6 +236,54 @@ private void createPipelines() throws RuntimeException {
     LOG.debug("BackgroundPipelineCreator createPipelines finished.");
   }
 
+  /**
+   * Returns replication configs eligible for background pipeline creation.
+   *
+   * <p>If the default replication config is invalid, this returns an empty
+   * list and skips pipeline creation to avoid guessing from raw config values.
+   * For EC-default clusters, this only returns RATIS/THREE when
+   * {@link ScmConfigKeys#OZONE_SCM_PIPELINE_CREATE_RATIS_THREE} is enabled.
+   */
+  @VisibleForTesting
+  List<ReplicationConfig> getReplicationConfigs(boolean autoCreateFactorOne) {
+    List<ReplicationConfig> list = new ArrayList<>();
+    ReplicationConfig defaultReplicationConfig = ScmUtils
+        .getDefaultReplicationConfig(conf, LOG,
+            BackgroundPipelineCreator.class.getSimpleName());
+    if (defaultReplicationConfig == null) {
+      LOG.warn("Skipping background pipeline creation: default replication "
+          + "config is invalid.");
+      return list;
+    }
+    // TODO: #CLUTIL Different replication factor may need to be supported
+    HddsProtos.ReplicationType type =
+        defaultReplicationConfig.getReplicationType();
+    if (type == EC && createRatisThreeForEcDefault) {
+      list.add(ReplicationConfig.fromProtoTypeAndFactor(RATIS,
+          ReplicationFactor.THREE));
+    }
+    if (type == EC) {
+      return list;
+    }
+
+    for (HddsProtos.ReplicationFactor factor
+        : HddsProtos.ReplicationFactor.values()) {
+      if (factor == ReplicationFactor.ZERO) {
+        continue; // Ignore it.
+      }
+      final ReplicationConfig replicationConfig =
+          ReplicationConfig.fromProtoTypeAndFactor(type, factor);
+      if (skipCreation(replicationConfig, autoCreateFactorOne)) {
+        // Skip this iteration for creating pipeline
+        continue;
+      }
+      if (!list.contains(replicationConfig)) {
+        list.add(replicationConfig);
+      }
+    }
+    return list;
+  }
+
   @Override
   public void notifyStatusChanged() {
     serviceLock.lock();

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java

@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.safemode;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmUtils;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
+
+/**
+ * Safe mode exit rule for EC-default clusters.
+ *
+ * <p>EC pipelines are ephemeral and created on demand. This rule ensures that
+ * at least {@code data + parity} healthy DataNodes are available before SCM
+ * exits safe mode for EC-default clusters.
+ *
+ * <p>For non-EC defaults this rule is a no-op.
+ */
+public class ECMinDataNodeSafeModeRule
+    extends SafeModeExitRule<NodeRegistrationContainerReport> {
+
+  private final boolean enabled;
+  private final int requiredDns;
+  private final String ecConfigLabel;
+  private final NodeManager nodeManager;
+  private final Set<DatanodeID> registeredDnSet;
+
+  public ECMinDataNodeSafeModeRule(EventQueue eventQueue,
+      ConfigurationSource conf,
+      NodeManager nodeManager,
+      SCMSafeModeManager safeModeManager) {
+    super(safeModeManager, eventQueue);
+    this.nodeManager = nodeManager;
+
+    ReplicationConfig defaultConfig = ScmUtils.getDefaultReplicationConfig(
+        conf, SCMSafeModeManager.getLogger(),
+        ECMinDataNodeSafeModeRule.class.getSimpleName());
+    if (defaultConfig != null
+        && defaultConfig.getReplicationType() == HddsProtos.ReplicationType.EC) {
+      ECReplicationConfig ecConfig = (ECReplicationConfig) defaultConfig;
+      this.requiredDns = ecConfig.getRequiredNodes();
+      this.ecConfigLabel = ecConfig.configFormat();
+      this.enabled = true;
+      this.registeredDnSet = new HashSet<>(Math.max(requiredDns * 2, 1));
+      SCMSafeModeManager.getLogger().info(
+          "ECMinDataNodeSafeModeRule enabled for default EC config {}. "
+              + "Required healthy DataNodes for safemode exit: {}.",
+          ecConfigLabel, requiredDns);
+    } else {
+      this.requiredDns = 0;
+      this.ecConfigLabel = "";
+      this.enabled = false;
+      this.registeredDnSet = new HashSet<>(0);
+      SCMSafeModeManager.getLogger().debug(
+          "ECMinDataNodeSafeModeRule disabled: default replication is not EC.");
+    }
+  }
+
+  @Override
+  protected TypedEvent<NodeRegistrationContainerReport> getEventType() {
+    return SCMEvents.NODE_REGISTRATION_CONT_REPORT;
+  }
+
+  @Override
+  protected synchronized boolean validate() {
+    if (!enabled) {
+      return true;
+    }
+    if (validateBasedOnReportProcessing()) {
+      return getRegisteredDns() >= requiredDns;
+    }
+    return nodeManager.getNodes(NodeStatus.inServiceHealthy()).size() >= requiredDns;
+  }
+
+  @Override
+  protected synchronized void process(NodeRegistrationContainerReport report) {
+    if (!enabled) {
+      return;
+    }
+    DatanodeID dnId = report.getDatanodeDetails().getID();
+    if (registeredDnSet.add(dnId)) {
+      if (scmInSafeMode()) {
+        SCMSafeModeManager.getLogger().info(
+            "SCM in safe mode. EC rule progress: {} of {} required "
+                + "DataNodes registered for EC {}.",
+            getRegisteredDns(), requiredDns, ecConfigLabel);
+      }
+    }
+  }
+
+  @Override
+  protected synchronized void cleanup() {
+    registeredDnSet.clear();
+  }
+
+  @Override
+  public synchronized String getStatusText() {
+    if (!enabled) {
+      return "ECMinDataNodeSafeModeRule is not applicable "
+          + "(default replication is not EC)";
+    }
+    return String.format(
+        "EC (%s) safemode: registered DataNodes (=%d) >= required DataNodes (=%d)",
+        ecConfigLabel, getRegisteredDns(), requiredDns);
+  }
+
+  @Override
+  public void refresh(boolean forceRefresh) {
+    // Nothing to refresh from SCM DB for this rule.
+  }
+
+  @VisibleForTesting
+  int getRequiredDns() {
+    return requiredDns;
+  }
+
+  @VisibleForTesting
+  synchronized int getRegisteredDns() {
+    return registeredDnSet.size();
+  }
+
+  boolean isEnabled() {
+    return enabled;
+  }
+}