From e13ffb80bf72b3dae3e968292cb654a997f3cbf5 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Wed, 29 Apr 2026 18:06:14 +0530 Subject: [PATCH 01/10] HDDS-15138. SCM safemode pipeline rules should honor default EC replication config. --- .../pipeline/BackgroundPipelineCreator.java | 82 ++++++++++++------- .../safemode/HealthyPipelineSafeModeRule.java | 54 ++++++------ .../OneReplicaPipelineSafeModeRule.java | 23 +++--- .../TestBackgroundPipelineCreator.java | 75 +++++++++++++++++ .../TestHealthyPipelineSafeModeRule.java | 50 +++++++++++ .../TestOneReplicaPipelineSafeModeRule.java | 37 +++++++++ 6 files changed, 255 insertions(+), 66 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java index 0aefbedbd43b..9fc928db75ab 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java @@ -25,6 +25,7 @@ import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.PRE_CHECK_COMPLETED; import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.time.Clock; @@ -199,47 +200,25 @@ private boolean skipCreation(ReplicationConfig replicationConfig, // used. return ((StandaloneReplicationConfig) replicationConfig) .getReplicationFactor() != ReplicationFactor.ONE; + } else if (replicationConfig.getReplicationType().equals(EC)) { + return false; } return true; } private void createPipelines() throws RuntimeException { - // TODO: #CLUTIL Different replication factor may need to be supported - HddsProtos.ReplicationType type = HddsProtos.ReplicationType.valueOf( - conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, - OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT)); - boolean autoCreateFactorOne = conf.getBoolean( - ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, + boolean autoCreateFactorOne = conf.getBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, ScmConfigKeys.OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE_DEFAULT); - List list = - new ArrayList<>(); - for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor - .values()) { - if (factor == ReplicationFactor.ZERO) { - continue; // Ignore it. - } - final ReplicationConfig replicationConfig; - if (type != EC) { - replicationConfig = - ReplicationConfig.fromProtoTypeAndFactor(type, factor); - } else if (factor == ReplicationFactor.ONE) { - replicationConfig = - ReplicationConfig.fromProtoTypeAndFactor(RATIS, factor); - } else { - continue; - } - if (skipCreation(replicationConfig, autoCreateFactorOne)) { - // Skip this iteration for creating pipeline - continue; - } - list.add(replicationConfig); + List list = getReplicationConfigs(autoCreateFactorOne); + if (list.isEmpty()) { + LOG.debug("No replication configs selected for background pipeline creation."); + return; } LoopingIterator it = new LoopingIterator(list); while (it.hasNext()) { - ReplicationConfig replicationConfig = - (ReplicationConfig) it.next(); + ReplicationConfig replicationConfig = (ReplicationConfig) it.next(); try { Pipeline pipeline = pipelineManager.createPipeline(replicationConfig); @@ -255,6 +234,49 @@ private void createPipelines() throws RuntimeException { LOG.debug("BackgroundPipelineCreator createPipelines finished."); } + @VisibleForTesting + List getReplicationConfigs(boolean autoCreateFactorOne) { + List list = new ArrayList<>(); + // TODO: #CLUTIL Different replication factor may need to be supported + HddsProtos.ReplicationType type = HddsProtos.ReplicationType.valueOf( + conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT)); + if (type == EC) { + try { + ReplicationConfig defaultConfig = ReplicationConfig.getDefault(conf); + if (defaultConfig.getReplicationType() == EC) { + list.add(defaultConfig); + } + } catch (IllegalArgumentException e) { + LOG.warn( + "Skipping EC pipeline creation due to invalid default EC " + "replication config. type={}, replication={}", + conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT), + conf.get(OzoneConfigKeys.OZONE_REPLICATION, OzoneConfigKeys.OZONE_REPLICATION_DEFAULT), e); + } + } + + for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor.values()) { + if (factor == ReplicationFactor.ZERO) { + continue; // Ignore it. + } + final ReplicationConfig replicationConfig; + if (type != EC) { + replicationConfig = ReplicationConfig.fromProtoTypeAndFactor(type, factor); + } else if (factor == ReplicationFactor.ONE) { + replicationConfig = ReplicationConfig.fromProtoTypeAndFactor(RATIS, factor); + } else { + continue; + } + if (skipCreation(replicationConfig, autoCreateFactorOne)) { + // Skip this iteration for creating pipeline + continue; + } + if (!list.contains(replicationConfig)) { + list.add(replicationConfig); + } + } + return list; + } + @Override public void notifyStatusChanged() { serviceLock.lock(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java index 3e590013c11f..be29ddd0aba2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java @@ -27,10 +27,9 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.node.NodeManager; @@ -66,6 +65,9 @@ public class HealthyPipelineSafeModeRule extends SafeModeExitRule { private final SCMContext scmContext; private final Set unProcessedPipelineSet = new HashSet<>(); private final NodeManager nodeManager; + private final ReplicationConfig targetReplicationConfig; + private final int targetRequiredNodes; + private final String targetReplicationLabel; HealthyPipelineSafeModeRule(EventQueue eventQueue, PipelineManager pipelineManager, SCMSafeModeManager manager, @@ -74,13 +76,15 @@ public class HealthyPipelineSafeModeRule extends SafeModeExitRule { this.pipelineManager = pipelineManager; this.scmContext = scmContext; this.nodeManager = nodeManager; + this.targetReplicationConfig = ReplicationConfig.getDefault(configuration); + this.targetRequiredNodes = targetReplicationConfig.getRequiredNodes(); + this.targetReplicationLabel = targetReplicationConfig.configFormat(); healthyPipelinesPercent = configuration.getDouble(HddsConfigKeys. HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT, HddsConfigKeys. HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT_DEFAULT); - // We only care about THREE replica pipeline minHealthyPipelines = getMinHealthyPipelines(configuration); Preconditions.checkArgument( @@ -97,8 +101,7 @@ private int getMinHealthyPipelines(ConfigurationSource config) { HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE_DEFAULT); - // We only care about THREE replica pipeline - return minDatanodes / HddsProtos.ReplicationFactor.THREE_VALUE; + return minDatanodes / targetRequiredNodes; } @@ -141,14 +144,12 @@ protected synchronized void process(Pipeline pipeline) { // datanode can send pipeline report again, or SCMPipelineManager will // create new pipelines. - // Only handle RATIS + 3-replica pipelines. - if (pipeline.getType() != HddsProtos.ReplicationType.RATIS || - ((RatisReplicationConfig) pipeline.getReplicationConfig()).getReplicationFactor() != - HddsProtos.ReplicationFactor.THREE) { + // Only handle pipelines matching the configured default replication. + if (!targetReplicationConfig.equals(pipeline.getReplicationConfig())) { Logger safeModeManagerLog = SCMSafeModeManager.getLogger(); if (safeModeManagerLog.isDebugEnabled()) { - safeModeManagerLog.debug("Skipping pipeline safemode report processing as Replication type isn't RATIS " + - "or replication factor isn't 3."); + safeModeManagerLog.debug("Skipping pipeline safemode report processing as replication config {} " + + "does not match target {}.", pipeline.getReplicationConfig(), targetReplicationConfig); } return; } @@ -161,9 +162,9 @@ protected synchronized void process(Pipeline pipeline) { } List pipelineDns = pipeline.getNodes(); - if (pipelineDns.size() != 3) { - LOG.warn("Only {} DNs reported this pipeline: {}, all 3 DNs should report the pipeline", pipelineDns.size(), - pipeline.getId()); + if (pipelineDns.size() != targetRequiredNodes) { + LOG.warn("Only {} DNs reported this pipeline: {}, all {} DNs should report the pipeline", + pipelineDns.size(), pipeline.getId(), targetRequiredNodes); return; } @@ -218,8 +219,7 @@ public synchronized void refresh(boolean forceRefresh) { private synchronized void initializeRule(boolean refresh) { unProcessedPipelineSet.addAll(pipelineManager.getPipelines( - RatisReplicationConfig.getInstance( - HddsProtos.ReplicationFactor.THREE), + targetReplicationConfig, Pipeline.PipelineState.OPEN).stream().map(Pipeline::getId) .collect(Collectors.toSet())); @@ -245,10 +245,11 @@ private synchronized void initializeRule(boolean refresh) { private boolean validateHealthyPipelineSafeModeRuleUsingPipelineManager() { // Query PipelineManager directly for healthy pipeline count List openPipelines = pipelineManager.getPipelines( - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + targetReplicationConfig, Pipeline.PipelineState.OPEN); - - LOG.debug("Found {} open RATIS/THREE pipelines", openPipelines.size()); + + LOG.debug("Found {} open {} pipelines", openPipelines.size(), + targetReplicationLabel); int pipelineCount = openPipelines.size(); healthyPipelineThresholdCount = Math.max(minHealthyPipelines, @@ -271,11 +272,11 @@ private boolean validateHealthyPipelineSafeModeRuleUsingPipelineManager() { } boolean isPipelineHealthy(Pipeline pipeline) { - // Verify pipeline has all 3 nodes + // Verify pipeline has all required nodes for target replication. List nodes = pipeline.getNodes(); - if (nodes.size() != 3) { - LOG.debug("Pipeline {} is not healthy: has {} nodes instead of 3", - pipeline.getId(), nodes.size()); + if (nodes.size() != targetRequiredNodes) { + LOG.debug("Pipeline {} is not healthy: has {} nodes instead of {}", + pipeline.getId(), nodes.size(), targetRequiredNodes); return false; } @@ -316,8 +317,9 @@ public synchronized int getHealthyPipelineThresholdCount() { @Override public String getStatusText() { String status = String.format( - "healthy Ratis/THREE pipelines (=%d) >= healthyPipelineThresholdCount" + - " (=%d)", getCurrentHealthyPipelineCount(), + "healthy %s pipelines (=%d) >= healthyPipelineThresholdCount" + + " (=%d)", targetReplicationLabel, + getCurrentHealthyPipelineCount(), getHealthyPipelineThresholdCount()); status = updateStatusTextWithSamplePipelines(status); return status; @@ -327,7 +329,7 @@ private synchronized String updateStatusTextWithSamplePipelines( String status) { if (validateBasedOnReportProcessing()) { List openPipelines = pipelineManager.getPipelines( - RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + targetReplicationConfig, Pipeline.PipelineState.OPEN); Set unhealthyPipelines = openPipelines.stream() diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java index 8b1fc593af38..e02a0b5e997b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java @@ -26,9 +26,8 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -58,6 +57,8 @@ public class OneReplicaPipelineSafeModeRule extends private int currentReportedPipelineCount = 0; private PipelineManager pipelineManager; private final double pipelinePercent; + private final ReplicationConfig targetReplicationConfig; + private final String targetReplicationLabel; public OneReplicaPipelineSafeModeRule(EventQueue eventQueue, PipelineManager pipelineManager, SCMSafeModeManager safeModeManager, ConfigurationSource configuration) { @@ -76,6 +77,8 @@ public OneReplicaPipelineSafeModeRule(EventQueue eventQueue, PipelineManager pip " value should be >= 0.0 and <= 1.0"); this.pipelineManager = pipelineManager; + this.targetReplicationConfig = ReplicationConfig.getDefault(configuration); + this.targetReplicationLabel = targetReplicationConfig.configFormat(); initializeRule(false); } @@ -108,8 +111,7 @@ protected synchronized void process(PipelineReportFromDatanode report) { continue; } - if (RatisReplicationConfig - .hasFactor(pipeline.getReplicationConfig(), ReplicationFactor.THREE) + if (targetReplicationConfig.equals(pipeline.getReplicationConfig()) && pipeline.isOpen() && !reportedPipelineIDSet.contains(pipeline.getId())) { if (oldPipelineIDSet.contains(pipeline.getId())) { @@ -152,8 +154,9 @@ Set getReportedPipelineIDSet() { @Override public String getStatusText() { String status = String.format( - "reported Ratis/THREE pipelines with at least one datanode (=%d) " - + ">= threshold (=%d)", getCurrentReportedPipelineCount(), + "reported %s pipelines with at least one datanode (=%d) " + + ">= threshold (=%d)", targetReplicationLabel, + getCurrentReportedPipelineCount(), getThresholdCount()); status = updateStatusTextWithSamplePipelines(status); return status; @@ -184,11 +187,11 @@ public synchronized void refresh(boolean forceRefresh) { } private void updateReportedPipelineSet() { - List openRatisPipelines = - pipelineManager.getPipelines(RatisReplicationConfig.getInstance(ReplicationFactor.THREE), + List openTargetPipelines = + pipelineManager.getPipelines(targetReplicationConfig, Pipeline.PipelineState.OPEN); - for (Pipeline pipeline : openRatisPipelines) { + for (Pipeline pipeline : openTargetPipelines) { PipelineID pipelineID = pipeline.getId(); if (!pipeline.getNodeSet().isEmpty() && oldPipelineIDSet.contains(pipelineID) @@ -202,7 +205,7 @@ private void updateReportedPipelineSet() { private void initializeRule(boolean refresh) { oldPipelineIDSet = pipelineManager.getPipelines( - RatisReplicationConfig.getInstance(ReplicationFactor.THREE), + targetReplicationConfig, Pipeline.PipelineState.OPEN) .stream().map(p -> p.getId()).collect(Collectors.toSet()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java new file mode 100644 index 000000000000..b38cf0c91e0e --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.pipeline; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import java.time.Clock; +import java.util.List; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.ha.SCMContext; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.junit.jupiter.api.Test; + +/** + * Tests for BackgroundPipelineCreator replication config selection. + */ +public class TestBackgroundPipelineCreator { + + @Test + public void testEcDefaultReplicationAddsEcPipelineConfig() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + + BackgroundPipelineCreator creator = new BackgroundPipelineCreator( + mock(PipelineManager.class), conf, mock(SCMContext.class), + Clock.systemUTC()); + + List configs = creator.getReplicationConfigs(false); + + assertTrue(configs.stream().anyMatch(c -> + c.getReplicationType() == HddsProtos.ReplicationType.EC)); + assertTrue(configs.contains(new ECReplicationConfig("rs-3-2-1024k"))); + } + + @Test + public void testRatisDefaultReplicationBehaviorUnchanged() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.RATIS.name()); + + BackgroundPipelineCreator creator = new BackgroundPipelineCreator( + mock(PipelineManager.class), conf, mock(SCMContext.class), + Clock.systemUTC()); + + List configs = creator.getReplicationConfigs(false); + + assertEquals(1, configs.size()); + assertTrue(RatisReplicationConfig.hasFactor(configs.get(0), + HddsProtos.ReplicationFactor.THREE)); + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index b1312512d567..4c1d87a39511 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -27,9 +27,11 @@ import java.time.Clock; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -44,11 +46,15 @@ import org.apache.hadoop.hdds.scm.ha.SCMServiceManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.junit.jupiter.api.Test; @@ -480,6 +486,50 @@ public void testPipelineIgnoredWhenDnIsUnhealthy() throws Exception { } } + @Test + public void testValidateWithEcDefaultReplicationConfig() throws Exception { + EventQueue eventQueue = new EventQueue(); + PipelineManagerImpl mockedPipelineManager = mock(PipelineManagerImpl.class); + SCMSafeModeManager mockedSafeModeManager = mock(SCMSafeModeManager.class); + SafeModeMetrics mockedMetrics = mock(SafeModeMetrics.class); + NodeManager mockedNodeManager = mock(NodeManager.class); + when(mockedSafeModeManager.getSafeModeMetrics()).thenReturn(mockedMetrics); + when(mockedSafeModeManager.getInSafeMode()).thenReturn(true); + + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setInt(HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, 0); + ReplicationConfig targetReplicationConfig = ReplicationConfig.getDefault(conf); + + DatanodeDetails dn1 = mock(DatanodeDetails.class); + DatanodeDetails dn2 = mock(DatanodeDetails.class); + DatanodeDetails dn3 = mock(DatanodeDetails.class); + DatanodeDetails dn4 = mock(DatanodeDetails.class); + DatanodeDetails dn5 = mock(DatanodeDetails.class); + List dnList = Arrays.asList(dn1, dn2, dn3, dn4, dn5); + for (DatanodeDetails dn : dnList) { + when(mockedNodeManager.getNodeStatus(dn)) + .thenReturn(NodeStatus.inServiceHealthy()); + } + + Pipeline ecPipeline = mock(Pipeline.class); + when(ecPipeline.getId()).thenReturn(PipelineID.randomId()); + when(ecPipeline.getNodes()).thenReturn(dnList); + when(mockedPipelineManager.getPipelines(targetReplicationConfig, + Pipeline.PipelineState.OPEN)).thenReturn(Arrays.asList(ecPipeline)); + + HealthyPipelineSafeModeRule localRule = + new HealthyPipelineSafeModeRule(eventQueue, mockedPipelineManager, + mockedSafeModeManager, conf, SCMContext.emptyContext(), + mockedNodeManager); + localRule.setValidateBasedOnReportProcessing(false); + + assertTrue(localRule.validate()); + assertTrue(localRule.getStatusText().contains("EC/3-2-1024k")); + } + private void firePipelineEvent(Pipeline pipeline, EventQueue eventQueue) { eventQueue.fireEvent(SCMEvents.OPEN_PIPELINE, pipeline); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java index e83b6e51a938..0a2948e808d3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.apache.ozone.test.TestClock; @@ -235,6 +236,42 @@ public void testOneReplicaPipelineRuleWithReportProcessingFalse() { assertTrue(localRule.getReportedPipelineIDSet().contains(pipelineID)); } + @Test + public void testOneReplicaPipelineRuleForEcDefaultReplication() { + EventQueue localEventQueue = new EventQueue(); + PipelineManager mockedPipelineManager = mock(PipelineManager.class); + SCMSafeModeManager mockedSafeModeManager = mock(SCMSafeModeManager.class); + SafeModeMetrics mockedMetrics = mock(SafeModeMetrics.class); + when(mockedSafeModeManager.getSafeModeMetrics()).thenReturn(mockedMetrics); + when(mockedSafeModeManager.getInSafeMode()).thenReturn(true); + + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + ReplicationConfig targetReplicationConfig = ReplicationConfig.getDefault(conf); + + PipelineID pipelineID = PipelineID.randomId(); + Pipeline mockedPipeline = mock(Pipeline.class); + when(mockedPipeline.getId()).thenReturn(pipelineID); + when(mockedPipeline.getNodeSet()) + .thenReturn(new java.util.HashSet<>( + java.util.Collections.singletonList(mock(DatanodeDetails.class)))); + + when(mockedPipelineManager.getPipelines(targetReplicationConfig, + Pipeline.PipelineState.OPEN)) + .thenReturn(java.util.Collections.singletonList(mockedPipeline)); + + OneReplicaPipelineSafeModeRule localRule = + new OneReplicaPipelineSafeModeRule(localEventQueue, mockedPipelineManager, + mockedSafeModeManager, conf); + localRule.setValidateBasedOnReportProcessing(false); + + assertTrue(localRule.validate()); + assertTrue(localRule.getReportedPipelineIDSet().contains(pipelineID)); + assertTrue(localRule.getStatusText().contains("EC/3-2-1024k")); + } + private void createPipelines(int count, HddsProtos.ReplicationFactor factor) throws Exception { for (int i = 0; i < count; i++) { From 61981c128efd1b44196e54baf7d755277b3db66f Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 12 May 2026 21:00:29 +0530 Subject: [PATCH 02/10] Fixed TestSCMSafeModeManager. --- .../hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index fdf38a7a67ca..45aa05af670e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -335,10 +335,10 @@ public void testSafeModeExitRuleWithPipelineAvailabilityCheck( assertEquals(1, scmSafeModeManager.getSafeModeMetrics().getScmInSafeMode().value()); if (healthyPipelinePercent > 0) { validateRuleStatus("HealthyPipelineSafeModeRule", - "healthy Ratis/THREE pipelines"); + "healthy RATIS/THREE pipelines"); } validateRuleStatus("OneReplicaPipelineSafeModeRule", - "reported Ratis/THREE pipelines with at least one datanode"); + "reported RATIS/THREE pipelines with at least one datanode"); testContainerThreshold(containers, 1.0); From 3877b05f6911c762bd034d1243e9a88468a30c87 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 16 Jun 2026 00:36:57 +0530 Subject: [PATCH 03/10] Add EC DN safemode rule and control RATIS/THREE background pipelines for EC-default clusters. --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 12 ++ .../pipeline/BackgroundPipelineCreator.java | 56 +++--- .../safemode/ECMinDataNodeSafeModeRule.java | 164 ++++++++++++++++++ .../safemode/HealthyPipelineSafeModeRule.java | 27 +-- .../OneReplicaPipelineSafeModeRule.java | 12 +- .../scm/safemode/SafeModeRuleFactory.java | 37 +++- .../TestBackgroundPipelineCreator.java | 33 +++- .../TestECMinDataNodeSafeModeRule.java | 87 ++++++++++ .../TestHealthyPipelineSafeModeRule.java | 50 ------ .../TestOneReplicaPipelineSafeModeRule.java | 40 +---- .../scm/safemode/TestSCMSafeModeManager.java | 76 +++++++- 11 files changed, 457 insertions(+), 137 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 2c44fa881c41..bd069407f819 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -454,6 +454,18 @@ public final class ScmConfigKeys { public static final boolean OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE_DEFAULT = true; + /** + * If true, BackgroundPipelineCreator will create RATIS/THREE pipelines even + * when the default replication is EC. This keeps RATIS write paths warm for + * mixed-workload clusters. If false, RATIS/THREE pipeline creation is + * skipped for EC-default clusters. + */ + public static final String OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE = + "ozone.scm.pipeline.creation.ec.ratis.three.enabled"; + + public static final boolean + OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE_DEFAULT = true; + public static final String OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR = "ozone.scm.block.deletion.per.dn.distribution.factor"; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java index 9fc928db75ab..719e88fbbc6b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java @@ -89,6 +89,7 @@ public class BackgroundPipelineCreator implements SCMService { private final AtomicBoolean running = new AtomicBoolean(false); private final long intervalInMillis; private final Clock clock; + private final boolean createRatisThreeForEcDefault; BackgroundPipelineCreator(PipelineManager pipelineManager, ConfigurationSource conf, SCMContext scmContext, Clock clock) { @@ -110,6 +111,9 @@ public class BackgroundPipelineCreator implements SCMService { ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + this.createRatisThreeForEcDefault = conf.getBoolean( + ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE_DEFAULT); threadName = scmContext.threadNamePrefix() + THREAD_NAME; } @@ -200,8 +204,6 @@ private boolean skipCreation(ReplicationConfig replicationConfig, // used. return ((StandaloneReplicationConfig) replicationConfig) .getReplicationFactor() != ReplicationFactor.ONE; - } else if (replicationConfig.getReplicationType().equals(EC)) { - return false; } return true; } @@ -237,35 +239,28 @@ private void createPipelines() throws RuntimeException { @VisibleForTesting List getReplicationConfigs(boolean autoCreateFactorOne) { List list = new ArrayList<>(); + ReplicationConfig defaultReplicationConfig = getDefaultReplicationConfig(); // TODO: #CLUTIL Different replication factor may need to be supported - HddsProtos.ReplicationType type = HddsProtos.ReplicationType.valueOf( - conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT)); + HddsProtos.ReplicationType type = defaultReplicationConfig != null + ? defaultReplicationConfig.getReplicationType() + : HddsProtos.ReplicationType.valueOf(conf.get( + OzoneConfigKeys.OZONE_REPLICATION_TYPE, + OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT)); if (type == EC) { - try { - ReplicationConfig defaultConfig = ReplicationConfig.getDefault(conf); - if (defaultConfig.getReplicationType() == EC) { - list.add(defaultConfig); - } - } catch (IllegalArgumentException e) { - LOG.warn( - "Skipping EC pipeline creation due to invalid default EC " + "replication config. type={}, replication={}", - conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT), - conf.get(OzoneConfigKeys.OZONE_REPLICATION, OzoneConfigKeys.OZONE_REPLICATION_DEFAULT), e); + if (createRatisThreeForEcDefault) { + list.add(ReplicationConfig.fromProtoTypeAndFactor(RATIS, + ReplicationFactor.THREE)); } + return list; } - for (HddsProtos.ReplicationFactor factor : HddsProtos.ReplicationFactor.values()) { + for (HddsProtos.ReplicationFactor factor + : HddsProtos.ReplicationFactor.values()) { if (factor == ReplicationFactor.ZERO) { continue; // Ignore it. } - final ReplicationConfig replicationConfig; - if (type != EC) { - replicationConfig = ReplicationConfig.fromProtoTypeAndFactor(type, factor); - } else if (factor == ReplicationFactor.ONE) { - replicationConfig = ReplicationConfig.fromProtoTypeAndFactor(RATIS, factor); - } else { - continue; - } + final ReplicationConfig replicationConfig = + ReplicationConfig.fromProtoTypeAndFactor(type, factor); if (skipCreation(replicationConfig, autoCreateFactorOne)) { // Skip this iteration for creating pipeline continue; @@ -277,6 +272,21 @@ List getReplicationConfigs(boolean autoCreateFactorOne) { return list; } + private ReplicationConfig getDefaultReplicationConfig() { + try { + return ReplicationConfig.getDefault(conf); + } catch (IllegalArgumentException e) { + LOG.warn("Ignoring invalid default replication config: type={}, " + + "replication={}.", + conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT), + conf.get(OzoneConfigKeys.OZONE_REPLICATION, + OzoneConfigKeys.OZONE_REPLICATION_DEFAULT), + e); + return null; + } + } + @Override public void notifyStatusChanged() { serviceLock.lock(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java new file mode 100644 index 000000000000..808792ed58f3 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.safemode; + +import com.google.common.annotations.VisibleForTesting; +import java.util.HashSet; +import java.util.Set; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.DatanodeID; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport; +import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.server.events.TypedEvent; + +/** + * Safe mode exit rule for EC-default clusters. + * + *

EC pipelines are ephemeral and created on demand. This rule ensures that + * at least {@code data + parity} healthy DataNodes are available before SCM + * exits safe mode for EC-default clusters. + * + *

For non-EC defaults this rule is a no-op. + */ +public class ECMinDataNodeSafeModeRule + extends SafeModeExitRule { + + private final boolean enabled; + private final int requiredDns; + private final String ecConfigLabel; + private final NodeManager nodeManager; + private final Set registeredDnSet; + private int registeredDns = 0; + + public ECMinDataNodeSafeModeRule(EventQueue eventQueue, + ConfigurationSource conf, + NodeManager nodeManager, + SCMSafeModeManager safeModeManager) { + super(safeModeManager, eventQueue); + this.nodeManager = nodeManager; + + ReplicationConfig defaultConfig = getDefaultReplicationConfig(conf); + if (defaultConfig != null + && defaultConfig.getReplicationType() == HddsProtos.ReplicationType.EC) { + ECReplicationConfig ecConfig = (ECReplicationConfig) defaultConfig; + this.requiredDns = ecConfig.getRequiredNodes(); + this.ecConfigLabel = ecConfig.configFormat(); + this.enabled = true; + this.registeredDnSet = new HashSet<>(Math.max(requiredDns * 2, 1)); + SCMSafeModeManager.getLogger().info( + "ECMinDataNodeSafeModeRule enabled for default EC config {}. " + + "Required healthy DataNodes for safemode exit: {}.", + ecConfigLabel, requiredDns); + } else { + this.requiredDns = 0; + this.ecConfigLabel = ""; + this.enabled = false; + this.registeredDnSet = new HashSet<>(0); + SCMSafeModeManager.getLogger().debug( + "ECMinDataNodeSafeModeRule disabled: default replication is not EC."); + } + } + + @Override + protected TypedEvent getEventType() { + return SCMEvents.NODE_REGISTRATION_CONT_REPORT; + } + + @Override + protected boolean validate() { + if (!enabled) { + return true; + } + if (validateBasedOnReportProcessing()) { + return registeredDns >= requiredDns; + } + return nodeManager.getNodes(NodeStatus.inServiceHealthy()).size() >= requiredDns; + } + + @Override + protected void process(NodeRegistrationContainerReport report) { + if (!enabled) { + return; + } + DatanodeID dnId = report.getDatanodeDetails().getID(); + if (registeredDnSet.add(dnId)) { + registeredDns = registeredDnSet.size(); + if (scmInSafeMode()) { + SCMSafeModeManager.getLogger().info( + "SCM in safe mode. EC rule progress: {} of {} required " + + "DataNodes registered for EC {}.", + registeredDns, requiredDns, ecConfigLabel); + } + } + } + + @Override + protected void cleanup() { + registeredDnSet.clear(); + } + + @Override + public String getStatusText() { + if (!enabled) { + return "ECMinDataNodeSafeModeRule is not applicable " + + "(default replication is not EC)"; + } + return String.format( + "EC (%s) safemode: registered DataNodes (=%d) >= required DataNodes (=%d)", + ecConfigLabel, registeredDns, requiredDns); + } + + @Override + public void refresh(boolean forceRefresh) { + // Nothing to refresh from SCM DB for this rule. + } + + @VisibleForTesting + int getRequiredDns() { + return requiredDns; + } + + @VisibleForTesting + int getRegisteredDns() { + return registeredDns; + } + + @VisibleForTesting + boolean isEnabled() { + return enabled; + } + + private static ReplicationConfig getDefaultReplicationConfig( + ConfigurationSource conf) { + try { + return ReplicationConfig.getDefault(conf); + } catch (IllegalArgumentException e) { + SCMSafeModeManager.getLogger().warn( + "Could not parse default replication config for " + + "ECMinDataNodeSafeModeRule; disabling EC rule.", + e); + return null; + } + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java index be29ddd0aba2..d6c301c32c5a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/HealthyPipelineSafeModeRule.java @@ -27,9 +27,11 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.node.NodeManager; @@ -65,9 +67,12 @@ public class HealthyPipelineSafeModeRule extends SafeModeExitRule { private final SCMContext scmContext; private final Set unProcessedPipelineSet = new HashSet<>(); private final NodeManager nodeManager; - private final ReplicationConfig targetReplicationConfig; - private final int targetRequiredNodes; - private final String targetReplicationLabel; + private final RatisReplicationConfig targetReplicationConfig = + RatisReplicationConfig.getInstance(ReplicationFactor.THREE); + private final int targetRequiredNodes = + HddsProtos.ReplicationFactor.THREE_VALUE; + private final String targetReplicationLabel = + targetReplicationConfig.configFormat(); HealthyPipelineSafeModeRule(EventQueue eventQueue, PipelineManager pipelineManager, SCMSafeModeManager manager, @@ -76,9 +81,6 @@ public class HealthyPipelineSafeModeRule extends SafeModeExitRule { this.pipelineManager = pipelineManager; this.scmContext = scmContext; this.nodeManager = nodeManager; - this.targetReplicationConfig = ReplicationConfig.getDefault(configuration); - this.targetRequiredNodes = targetReplicationConfig.getRequiredNodes(); - this.targetReplicationLabel = targetReplicationConfig.configFormat(); healthyPipelinesPercent = configuration.getDouble(HddsConfigKeys. HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT, @@ -101,7 +103,7 @@ private int getMinHealthyPipelines(ConfigurationSource config) { HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE_DEFAULT); - return minDatanodes / targetRequiredNodes; + return minDatanodes / HddsProtos.ReplicationFactor.THREE_VALUE; } @@ -144,12 +146,12 @@ protected synchronized void process(Pipeline pipeline) { // datanode can send pipeline report again, or SCMPipelineManager will // create new pipelines. - // Only handle pipelines matching the configured default replication. if (!targetReplicationConfig.equals(pipeline.getReplicationConfig())) { Logger safeModeManagerLog = SCMSafeModeManager.getLogger(); if (safeModeManagerLog.isDebugEnabled()) { - safeModeManagerLog.debug("Skipping pipeline safemode report processing as replication config {} " + - "does not match target {}.", pipeline.getReplicationConfig(), targetReplicationConfig); + safeModeManagerLog.debug("Skipping pipeline safemode report processing" + + " as replication config {} does not match target {}.", + pipeline.getReplicationConfig(), targetReplicationConfig); } return; } @@ -163,7 +165,8 @@ protected synchronized void process(Pipeline pipeline) { List pipelineDns = pipeline.getNodes(); if (pipelineDns.size() != targetRequiredNodes) { - LOG.warn("Only {} DNs reported this pipeline: {}, all {} DNs should report the pipeline", + LOG.warn("Only {} DNs reported this pipeline: {}, all {} DNs should " + + "report the pipeline", pipelineDns.size(), pipeline.getId(), targetRequiredNodes); return; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java index e02a0b5e997b..227d7ddb47b5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/OneReplicaPipelineSafeModeRule.java @@ -26,8 +26,9 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; @@ -57,8 +58,8 @@ public class OneReplicaPipelineSafeModeRule extends private int currentReportedPipelineCount = 0; private PipelineManager pipelineManager; private final double pipelinePercent; - private final ReplicationConfig targetReplicationConfig; - private final String targetReplicationLabel; + private final RatisReplicationConfig targetReplicationConfig = + RatisReplicationConfig.getInstance(ReplicationFactor.THREE); public OneReplicaPipelineSafeModeRule(EventQueue eventQueue, PipelineManager pipelineManager, SCMSafeModeManager safeModeManager, ConfigurationSource configuration) { @@ -77,8 +78,6 @@ public OneReplicaPipelineSafeModeRule(EventQueue eventQueue, PipelineManager pip " value should be >= 0.0 and <= 1.0"); this.pipelineManager = pipelineManager; - this.targetReplicationConfig = ReplicationConfig.getDefault(configuration); - this.targetReplicationLabel = targetReplicationConfig.configFormat(); initializeRule(false); } @@ -155,7 +154,8 @@ Set getReportedPipelineIDSet() { public String getStatusText() { String status = String.format( "reported %s pipelines with at least one datanode (=%d) " - + ">= threshold (=%d)", targetReplicationLabel, + + ">= threshold (=%d)", + targetReplicationConfig.configFormat(), getCurrentReportedPipelineCount(), getThresholdCount()); status = updateStatusTextWithSamplePipelines(status); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java index 398eb19b56ec..06b1545f5ede 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java @@ -19,7 +19,10 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.SCMHAManager; @@ -75,10 +78,13 @@ private void loadRules(SCMSafeModeManager safeModeManager) { config, containerManager, safeModeManager); SafeModeExitRule datanodeRule = new DataNodeSafeModeRule(eventQueue, config, nodeManager, safeModeManager); + SafeModeExitRule ecMinDnRule = new ECMinDataNodeSafeModeRule(eventQueue, + config, nodeManager, safeModeManager); safeModeRules.add(ratisContainerRule); safeModeRules.add(ecContainerRule); safeModeRules.add(datanodeRule); + safeModeRules.add(ecMinDnRule); preCheckRules.add(datanodeRule); @@ -93,14 +99,41 @@ private void loadRules(SCMSafeModeManager safeModeManager) { } if (pipelineManager != null) { - safeModeRules.add(new HealthyPipelineSafeModeRule(eventQueue, pipelineManager, - safeModeManager, config, scmContext, nodeManager)); + if (shouldEnableHealthyPipelineRule()) { + safeModeRules.add(new HealthyPipelineSafeModeRule(eventQueue, + pipelineManager, safeModeManager, config, scmContext, nodeManager)); + } else { + SCMSafeModeManager.getLogger().info("HealthyPipelineSafeModeRule is " + + "disabled for EC-default cluster because " + + "{} is false.", + ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE); + } safeModeRules.add(new OneReplicaPipelineSafeModeRule(eventQueue, pipelineManager, safeModeManager, config)); } } + private boolean shouldEnableHealthyPipelineRule() { + ReplicationConfig defaultReplicationConfig; + try { + defaultReplicationConfig = ReplicationConfig.getDefault(config); + } catch (IllegalArgumentException e) { + SCMSafeModeManager.getLogger().warn("Falling back to enabling " + + "HealthyPipelineSafeModeRule because default replication config " + + "could not be parsed.", e); + return true; + } + + if (defaultReplicationConfig.getReplicationType() + != HddsProtos.ReplicationType.EC) { + return true; + } + + return config.getBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE_DEFAULT); + } + public static synchronized SafeModeRuleFactory getInstance() { if (instance != null) { return instance; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java index b38cf0c91e0e..7c3f076be02a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java @@ -18,16 +18,17 @@ package org.apache.hadoop.hdds.scm.pipeline; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import java.time.Clock; import java.util.List; -import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.junit.jupiter.api.Test; @@ -38,11 +39,14 @@ public class TestBackgroundPipelineCreator { @Test - public void testEcDefaultReplicationAddsEcPipelineConfig() throws Exception { + public void testEcDefaultReplicationWithoutRatisThreeFlagCreatesNoPipelines() + throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + false); BackgroundPipelineCreator creator = new BackgroundPipelineCreator( mock(PipelineManager.class), conf, mock(SCMContext.class), @@ -50,9 +54,30 @@ public void testEcDefaultReplicationAddsEcPipelineConfig() throws Exception { List configs = creator.getReplicationConfigs(false); - assertTrue(configs.stream().anyMatch(c -> + assertTrue(configs.isEmpty()); + } + + @Test + public void testEcDefaultReplicationWithRatisThreeFlag() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + true); + + BackgroundPipelineCreator creator = new BackgroundPipelineCreator( + mock(PipelineManager.class), conf, mock(SCMContext.class), + Clock.systemUTC()); + + List configs = creator.getReplicationConfigs(false); + + assertEquals(1, configs.size()); + assertTrue(configs.stream() + .anyMatch(c -> RatisReplicationConfig.hasFactor(c, + HddsProtos.ReplicationFactor.THREE))); + assertFalse(configs.stream().anyMatch(c -> c.getReplicationType() == HddsProtos.ReplicationType.EC)); - assertTrue(configs.contains(new ECReplicationConfig("rs-3-2-1024k"))); } @Test diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java new file mode 100644 index 000000000000..c1a61a6e2c49 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.safemode; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link ECMinDataNodeSafeModeRule}. + */ +public class TestECMinDataNodeSafeModeRule { + + @Test + public void testDisabledForNonEcDefault() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.RATIS.name()); + + NodeManager nodeManager = mock(NodeManager.class); + SCMSafeModeManager safeModeManager = mock(SCMSafeModeManager.class); + when(safeModeManager.getSafeModeMetrics()).thenReturn(mock(SafeModeMetrics.class)); + + ECMinDataNodeSafeModeRule rule = new ECMinDataNodeSafeModeRule( + new EventQueue(), conf, nodeManager, safeModeManager); + + assertFalse(rule.isEnabled()); + assertTrue(rule.validate()); + } + + @Test + public void testEnabledForEcDefaultAndUsesRequiredNodeCount() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + + List enoughDns = new ArrayList<>(); + List insufficientDns = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + enoughDns.add(mock(DatanodeDetails.class)); + if (i < 4) { + insufficientDns.add(mock(DatanodeDetails.class)); + } + } + + NodeManager nodeManager = mock(NodeManager.class); + when(nodeManager.getNodes(any())).thenReturn(enoughDns, insufficientDns); + SCMSafeModeManager safeModeManager = mock(SCMSafeModeManager.class); + when(safeModeManager.getSafeModeMetrics()).thenReturn(mock(SafeModeMetrics.class)); + + ECMinDataNodeSafeModeRule rule = new ECMinDataNodeSafeModeRule( + new EventQueue(), conf, nodeManager, safeModeManager); + rule.setValidateBasedOnReportProcessing(false); + + assertTrue(rule.isEnabled()); + assertTrue(rule.validate()); + assertFalse(rule.validate()); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index 4c1d87a39511..b1312512d567 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -27,11 +27,9 @@ import java.time.Clock; import java.time.ZoneOffset; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -46,15 +44,11 @@ import org.apache.hadoop.hdds.scm.ha.SCMServiceManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; -import org.apache.hadoop.hdds.scm.node.NodeManager; -import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.junit.jupiter.api.Test; @@ -486,50 +480,6 @@ public void testPipelineIgnoredWhenDnIsUnhealthy() throws Exception { } } - @Test - public void testValidateWithEcDefaultReplicationConfig() throws Exception { - EventQueue eventQueue = new EventQueue(); - PipelineManagerImpl mockedPipelineManager = mock(PipelineManagerImpl.class); - SCMSafeModeManager mockedSafeModeManager = mock(SCMSafeModeManager.class); - SafeModeMetrics mockedMetrics = mock(SafeModeMetrics.class); - NodeManager mockedNodeManager = mock(NodeManager.class); - when(mockedSafeModeManager.getSafeModeMetrics()).thenReturn(mockedMetrics); - when(mockedSafeModeManager.getInSafeMode()).thenReturn(true); - - OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, - HddsProtos.ReplicationType.EC.name()); - conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setInt(HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, 0); - ReplicationConfig targetReplicationConfig = ReplicationConfig.getDefault(conf); - - DatanodeDetails dn1 = mock(DatanodeDetails.class); - DatanodeDetails dn2 = mock(DatanodeDetails.class); - DatanodeDetails dn3 = mock(DatanodeDetails.class); - DatanodeDetails dn4 = mock(DatanodeDetails.class); - DatanodeDetails dn5 = mock(DatanodeDetails.class); - List dnList = Arrays.asList(dn1, dn2, dn3, dn4, dn5); - for (DatanodeDetails dn : dnList) { - when(mockedNodeManager.getNodeStatus(dn)) - .thenReturn(NodeStatus.inServiceHealthy()); - } - - Pipeline ecPipeline = mock(Pipeline.class); - when(ecPipeline.getId()).thenReturn(PipelineID.randomId()); - when(ecPipeline.getNodes()).thenReturn(dnList); - when(mockedPipelineManager.getPipelines(targetReplicationConfig, - Pipeline.PipelineState.OPEN)).thenReturn(Arrays.asList(ecPipeline)); - - HealthyPipelineSafeModeRule localRule = - new HealthyPipelineSafeModeRule(eventQueue, mockedPipelineManager, - mockedSafeModeManager, conf, SCMContext.emptyContext(), - mockedNodeManager); - localRule.setValidateBasedOnReportProcessing(false); - - assertTrue(localRule.validate()); - assertTrue(localRule.getStatusText().contains("EC/3-2-1024k")); - } - private void firePipelineEvent(Pipeline pipeline, EventQueue eventQueue) { eventQueue.fireEvent(SCMEvents.OPEN_PIPELINE, pipeline); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java index 0a2948e808d3..801b5351324e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java @@ -31,7 +31,6 @@ import java.util.Map; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.RatisReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -57,7 +56,6 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.apache.ozone.test.TestClock; @@ -218,7 +216,7 @@ public void testOneReplicaPipelineRuleWithReportProcessingFalse() { java.util.Collections.singletonList(mock(DatanodeDetails.class)))); when(mockedPipelineManager.getPipelines( - Mockito.any(ReplicationConfig.class), + Mockito.any(), Mockito.eq(Pipeline.PipelineState.OPEN))) .thenReturn(java.util.Collections.singletonList(mockedPipeline)); @@ -236,42 +234,6 @@ public void testOneReplicaPipelineRuleWithReportProcessingFalse() { assertTrue(localRule.getReportedPipelineIDSet().contains(pipelineID)); } - @Test - public void testOneReplicaPipelineRuleForEcDefaultReplication() { - EventQueue localEventQueue = new EventQueue(); - PipelineManager mockedPipelineManager = mock(PipelineManager.class); - SCMSafeModeManager mockedSafeModeManager = mock(SCMSafeModeManager.class); - SafeModeMetrics mockedMetrics = mock(SafeModeMetrics.class); - when(mockedSafeModeManager.getSafeModeMetrics()).thenReturn(mockedMetrics); - when(mockedSafeModeManager.getInSafeMode()).thenReturn(true); - - OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, - HddsProtos.ReplicationType.EC.name()); - conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - ReplicationConfig targetReplicationConfig = ReplicationConfig.getDefault(conf); - - PipelineID pipelineID = PipelineID.randomId(); - Pipeline mockedPipeline = mock(Pipeline.class); - when(mockedPipeline.getId()).thenReturn(pipelineID); - when(mockedPipeline.getNodeSet()) - .thenReturn(new java.util.HashSet<>( - java.util.Collections.singletonList(mock(DatanodeDetails.class)))); - - when(mockedPipelineManager.getPipelines(targetReplicationConfig, - Pipeline.PipelineState.OPEN)) - .thenReturn(java.util.Collections.singletonList(mockedPipeline)); - - OneReplicaPipelineSafeModeRule localRule = - new OneReplicaPipelineSafeModeRule(localEventQueue, mockedPipelineManager, - mockedSafeModeManager, conf); - localRule.setValidateBasedOnReportProcessing(false); - - assertTrue(localRule.validate()); - assertTrue(localRule.getReportedPipelineIDSet().contains(pipelineID)); - assertTrue(localRule.getStatusText().contains("EC/3-2-1024k")); - } - private void createPipelines(int count, HddsProtos.ReplicationFactor factor) throws Exception { for (int i = 0; i < count; i++) { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index 45aa05af670e..3b6f5532afd1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -47,6 +48,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl; @@ -72,6 +74,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -410,7 +413,7 @@ private void validateRuleStatus(String safeModeRule, String stringToMatch) { if (entry.getKey().equals(safeModeRule)) { Pair value = entry.getValue(); assertEquals(false, value.getLeft()); - assertThat(value.getRight()).contains(stringToMatch); + assertThat(value.getRight()).containsIgnoringCase(stringToMatch); } } } @@ -751,6 +754,77 @@ public void testSafeModePipelineExitRule() throws Exception { pipelineManager.close(); } + @Test + public void testEcDefaultDisablesHealthyPipelineRuleWhenRatisThreeDisabled() { + OzoneConfiguration conf = new OzoneConfiguration(config); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + false); + + MockNodeManager mockNodeManager = new MockNodeManager(true, 5); + PipelineManager pipelineManager = mock(PipelineManager.class); + when(pipelineManager.getPipelines(any(), any())) + .thenReturn(Collections.emptyList()); + when(pipelineManager.getPipelines()) + .thenReturn(Collections.emptyList()); + + ContainerManager containerManager = mock(ContainerManager.class); + when(containerManager.getContainers(ReplicationType.RATIS)) + .thenReturn(Collections.emptyList()); + when(containerManager.getContainers(ReplicationType.EC)) + .thenReturn(Collections.emptyList()); + when(containerManager.getContainers()) + .thenReturn(Collections.emptyList()); + + scmSafeModeManager = new SCMSafeModeManager(conf, mockNodeManager, + pipelineManager, containerManager, serviceManager, queue, scmContext); + scmSafeModeManager.start(); + + assertThat(SafeModeRuleFactory.getInstance() + .getSafeModeRule(HealthyPipelineSafeModeRule.class)).isNull(); + ECMinDataNodeSafeModeRule ecMinDnRule = SafeModeRuleFactory.getInstance() + .getSafeModeRule(ECMinDataNodeSafeModeRule.class); + assertThat(ecMinDnRule).isNotNull(); + assertThat(ecMinDnRule.isEnabled()).isTrue(); + assertThat(ecMinDnRule.getRequiredDns()).isEqualTo(5); + } + + @Test + public void testEcDefaultKeepsHealthyPipelineRuleWhenRatisThreeEnabled() { + OzoneConfiguration conf = new OzoneConfiguration(config); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + true); + + MockNodeManager mockNodeManager = new MockNodeManager(true, 5); + PipelineManager pipelineManager = mock(PipelineManager.class); + when(pipelineManager.getPipelines(any(), any())) + .thenReturn(Collections.emptyList()); + when(pipelineManager.getPipelines()) + .thenReturn(Collections.emptyList()); + + ContainerManager containerManager = mock(ContainerManager.class); + when(containerManager.getContainers(ReplicationType.RATIS)) + .thenReturn(Collections.emptyList()); + when(containerManager.getContainers(ReplicationType.EC)) + .thenReturn(Collections.emptyList()); + when(containerManager.getContainers()) + .thenReturn(Collections.emptyList()); + + scmSafeModeManager = new SCMSafeModeManager(conf, mockNodeManager, + pipelineManager, containerManager, serviceManager, queue, scmContext); + scmSafeModeManager.start(); + + assertThat(SafeModeRuleFactory.getInstance() + .getSafeModeRule(HealthyPipelineSafeModeRule.class)).isNotNull(); + assertThat(SafeModeRuleFactory.getInstance() + .getSafeModeRule(ECMinDataNodeSafeModeRule.class)).isNotNull(); + } + @Test public void testPipelinesNotCreatedUntilPreCheckPasses() throws Exception { int numOfDns = 5; From 1923eeef1bdbb03d99d23a0eebd15a0e06b27632 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 16 Jun 2026 02:10:59 +0530 Subject: [PATCH 04/10] Addressed test failures. --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 9 +++++++++ .../hdds/scm/safemode/TestSafeModeRuleFactory.java | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index e159eb6948b8..cba646155164 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1710,6 +1710,15 @@ If enabled, SCM will auto create RATIS factor ONE pipeline. + + ozone.scm.pipeline.creation.ec.ratis.three.enabled + true + OZONE, SCM, PIPELINE + + If enabled, SCM creates RATIS/THREE background pipelines even when the + default replication type is EC. + + hdds.scm.safemode.threshold.pct 0.99 diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java index f795a6c57628..d6f22e969031 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java @@ -56,7 +56,7 @@ public void testLoadedSafeModeRules() { // as the rules are hardcoded in SafeModeRuleFactory. // This will be fixed once we load rules using annotation. - assertEquals(5, factory.getSafeModeRules().size(), + assertEquals(6, factory.getSafeModeRules().size(), "The total safemode rules count doesn't match"); } From f4805ebd56ffa8db922785704b4e3557d4bdb0e4 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 16 Jun 2026 14:25:34 +0530 Subject: [PATCH 05/10] Addressed comments. --- .../safemode/ECMinDataNodeSafeModeRule.java | 13 ++-- .../TestSCMSafeModeWithPipelineRules.java | 65 +++++++++++++++++-- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java index 808792ed58f3..09d167b64b9c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java @@ -86,7 +86,7 @@ protected TypedEvent getEventType() { } @Override - protected boolean validate() { + protected synchronized boolean validate() { if (!enabled) { return true; } @@ -97,7 +97,7 @@ protected boolean validate() { } @Override - protected void process(NodeRegistrationContainerReport report) { + protected synchronized void process(NodeRegistrationContainerReport report) { if (!enabled) { return; } @@ -114,12 +114,12 @@ protected void process(NodeRegistrationContainerReport report) { } @Override - protected void cleanup() { + protected synchronized void cleanup() { registeredDnSet.clear(); } @Override - public String getStatusText() { + public synchronized String getStatusText() { if (!enabled) { return "ECMinDataNodeSafeModeRule is not applicable " + "(default replication is not EC)"; @@ -139,11 +139,6 @@ int getRequiredDns() { return requiredDns; } - @VisibleForTesting - int getRegisteredDns() { - return registeredDns; - } - @VisibleForTesting boolean isEnabled() { return enabled; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java index 135a8389c349..fe25b8cec5d6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.safemode; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; @@ -31,9 +32,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -43,6 +46,10 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -58,6 +65,17 @@ public class TestSCMSafeModeWithPipelineRules { public void setup(int numDatanodes) throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); + configureTestCluster(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(numDatanodes) + .build(); + cluster.waitForClusterToBeReady(); + StorageContainerManager scm = cluster.getStorageContainerManager(); + pipelineManager = scm.getPipelineManager(); + } + + private static void configureTestCluster(OzoneConfiguration conf) { conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100, TimeUnit.MILLISECONDS); conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "10s"); @@ -69,13 +87,6 @@ public void setup(int numDatanodes) throws Exception { conf.setBoolean(ScmConfigKeys.OZONE_SCM_DATANODE_DISALLOW_SAME_PEERS, true); conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY, SCMContainerPlacementCapacity.class, PlacementPolicy.class); - - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(numDatanodes) - .build(); - cluster.waitForClusterToBeReady(); - StorageContainerManager scm = cluster.getStorageContainerManager(); - pipelineManager = scm.getPipelineManager(); } @Test @@ -156,6 +167,46 @@ void testScmSafeMode() throws Exception { GenericTestUtils.waitFor(replicationManager::isRunning, 1000, 60000); } + @Test + void testSafeModeExitAfterScmRestartWithMixedEcAndRatisThreeKeys() + throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + configureTestCluster(conf); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + true); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) + .build(); + cluster.waitForClusterToBeReady(); + pipelineManager = cluster.getStorageContainerManager().getPipelineManager(); + waitForRatis3NodePipelines(1); + + try (OzoneClient client = cluster.newClient()) { + OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client); + TestDataUtil.createKey(bucket, "ec-key", + new ECReplicationConfig("rs-3-2-1024k"), + "ec-data".getBytes(UTF_8)); + TestDataUtil.createKey(bucket, "ratis3-key", + RatisReplicationConfig.getInstance(ReplicationFactor.THREE), + "ratis-data".getBytes(UTF_8)); + } + + cluster.restartStorageContainerManager(false); + SCMSafeModeManager scmSafeModeManager = + cluster.getStorageContainerManager().getScmSafeModeManager(); + final ECMinDataNodeSafeModeRule ecRule = SafeModeRuleFactory.getInstance() + .getSafeModeRule(ECMinDataNodeSafeModeRule.class); + + assertTrue(ecRule.isEnabled()); + GenericTestUtils.waitFor(ecRule::validate, 1000, 60000); + GenericTestUtils.waitFor(() -> !scmSafeModeManager.getInSafeMode(), 1000, + 60000); + } + @AfterEach public void tearDown() { if (cluster != null) { From abf6b27499903c7278baf4f060e58ff9629d3c69 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Tue, 16 Jun 2026 16:12:38 +0530 Subject: [PATCH 06/10] Addressed comments. --- .../pipeline/BackgroundPipelineCreator.java | 12 ++++--- .../safemode/ECMinDataNodeSafeModeRule.java | 13 +++++--- .../scm/safemode/SafeModeRuleFactory.java | 31 ++++++++++++------- .../TestBackgroundPipelineCreator.java | 14 +++++++++ .../scm/safemode/TestSCMSafeModeManager.java | 4 +++ 5 files changed, 53 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java index 719e88fbbc6b..15f80659aabe 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java @@ -240,12 +240,14 @@ private void createPipelines() throws RuntimeException { List getReplicationConfigs(boolean autoCreateFactorOne) { List list = new ArrayList<>(); ReplicationConfig defaultReplicationConfig = getDefaultReplicationConfig(); + if (defaultReplicationConfig == null) { + LOG.warn("Skipping background pipeline creation: default replication " + + "config is invalid."); + return list; + } // TODO: #CLUTIL Different replication factor may need to be supported - HddsProtos.ReplicationType type = defaultReplicationConfig != null - ? defaultReplicationConfig.getReplicationType() - : HddsProtos.ReplicationType.valueOf(conf.get( - OzoneConfigKeys.OZONE_REPLICATION_TYPE, - OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT)); + HddsProtos.ReplicationType type = + defaultReplicationConfig.getReplicationType(); if (type == EC) { if (createRatisThreeForEcDefault) { list.add(ReplicationConfig.fromProtoTypeAndFactor(RATIS, diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java index 09d167b64b9c..af80053c3db3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java @@ -49,7 +49,6 @@ public class ECMinDataNodeSafeModeRule private final String ecConfigLabel; private final NodeManager nodeManager; private final Set registeredDnSet; - private int registeredDns = 0; public ECMinDataNodeSafeModeRule(EventQueue eventQueue, ConfigurationSource conf, @@ -91,7 +90,7 @@ protected synchronized boolean validate() { return true; } if (validateBasedOnReportProcessing()) { - return registeredDns >= requiredDns; + return getRegisteredDns() >= requiredDns; } return nodeManager.getNodes(NodeStatus.inServiceHealthy()).size() >= requiredDns; } @@ -103,12 +102,11 @@ protected synchronized void process(NodeRegistrationContainerReport report) { } DatanodeID dnId = report.getDatanodeDetails().getID(); if (registeredDnSet.add(dnId)) { - registeredDns = registeredDnSet.size(); if (scmInSafeMode()) { SCMSafeModeManager.getLogger().info( "SCM in safe mode. EC rule progress: {} of {} required " + "DataNodes registered for EC {}.", - registeredDns, requiredDns, ecConfigLabel); + getRegisteredDns(), requiredDns, ecConfigLabel); } } } @@ -126,7 +124,7 @@ public synchronized String getStatusText() { } return String.format( "EC (%s) safemode: registered DataNodes (=%d) >= required DataNodes (=%d)", - ecConfigLabel, registeredDns, requiredDns); + ecConfigLabel, getRegisteredDns(), requiredDns); } @Override @@ -139,6 +137,11 @@ int getRequiredDns() { return requiredDns; } + @VisibleForTesting + synchronized int getRegisteredDns() { + return registeredDnSet.size(); + } + @VisibleForTesting boolean isEnabled() { return enabled; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java index 06b1545f5ede..1e91aa881974 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java @@ -99,30 +99,39 @@ private void loadRules(SCMSafeModeManager safeModeManager) { } if (pipelineManager != null) { - if (shouldEnableHealthyPipelineRule()) { + if (shouldEnableRatisThreePipelineRules()) { safeModeRules.add(new HealthyPipelineSafeModeRule(eventQueue, pipelineManager, safeModeManager, config, scmContext, nodeManager)); + safeModeRules.add(new OneReplicaPipelineSafeModeRule(eventQueue, pipelineManager, + safeModeManager, config)); } else { - SCMSafeModeManager.getLogger().info("HealthyPipelineSafeModeRule is " - + "disabled for EC-default cluster because " - + "{} is false.", + SCMSafeModeManager.getLogger().info( + "RATIS/THREE pipeline safemode rules are disabled because " + + "{} is false for an EC-default cluster or the default " + + "replication config is invalid.", ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE); } - safeModeRules.add(new OneReplicaPipelineSafeModeRule(eventQueue, pipelineManager, - safeModeManager, config)); } } - private boolean shouldEnableHealthyPipelineRule() { + /** + * Returns true when RATIS/THREE pipeline safemode rules should be active. + * For EC-default clusters, these rules are only meaningful when RATIS/THREE + * background pipeline creation is also enabled (same flag); if no + * RATIS/THREE pipelines are created, requiring them in safemode would block + * safemode exit. + */ + private boolean shouldEnableRatisThreePipelineRules() { ReplicationConfig defaultReplicationConfig; try { defaultReplicationConfig = ReplicationConfig.getDefault(config); } catch (IllegalArgumentException e) { - SCMSafeModeManager.getLogger().warn("Falling back to enabling " - + "HealthyPipelineSafeModeRule because default replication config " - + "could not be parsed.", e); - return true; + SCMSafeModeManager.getLogger().warn( + "Disabling RATIS/THREE pipeline safemode rules because default " + + "replication config could not be parsed.", + e); + return false; } if (defaultReplicationConfig.getReplicationType() diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java index 7c3f076be02a..2bbe9fd8c38c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java @@ -97,4 +97,18 @@ public void testRatisDefaultReplicationBehaviorUnchanged() throws Exception { HddsProtos.ReplicationFactor.THREE)); } + @Test + public void testInvalidDefaultReplicationConfigCreatesNoPipelines() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.RATIS.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "invalid-replication-value"); + + BackgroundPipelineCreator creator = + new BackgroundPipelineCreator(mock(PipelineManager.class), conf, mock(SCMContext.class), Clock.systemUTC()); + + List configs = creator.getReplicationConfigs(false); + + assertTrue(configs.isEmpty()); + } + } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index 3b6f5532afd1..7c9140fba720 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -784,6 +784,8 @@ public void testEcDefaultDisablesHealthyPipelineRuleWhenRatisThreeDisabled() { assertThat(SafeModeRuleFactory.getInstance() .getSafeModeRule(HealthyPipelineSafeModeRule.class)).isNull(); + assertThat(SafeModeRuleFactory.getInstance() + .getSafeModeRule(OneReplicaPipelineSafeModeRule.class)).isNull(); ECMinDataNodeSafeModeRule ecMinDnRule = SafeModeRuleFactory.getInstance() .getSafeModeRule(ECMinDataNodeSafeModeRule.class); assertThat(ecMinDnRule).isNotNull(); @@ -821,6 +823,8 @@ public void testEcDefaultKeepsHealthyPipelineRuleWhenRatisThreeEnabled() { assertThat(SafeModeRuleFactory.getInstance() .getSafeModeRule(HealthyPipelineSafeModeRule.class)).isNotNull(); + assertThat(SafeModeRuleFactory.getInstance() + .getSafeModeRule(OneReplicaPipelineSafeModeRule.class)).isNotNull(); assertThat(SafeModeRuleFactory.getInstance() .getSafeModeRule(ECMinDataNodeSafeModeRule.class)).isNotNull(); } From 742f02493a5a3a8dc1c89ef73141139a5cf05a3a Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Wed, 17 Jun 2026 14:49:31 +0530 Subject: [PATCH 07/10] Fixed TestSCMSafeModeWithPipelineRules. --- .../scm/safemode/TestSCMSafeModeWithPipelineRules.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java index fe25b8cec5d6..27dd1fad089b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java @@ -202,9 +202,12 @@ void testSafeModeExitAfterScmRestartWithMixedEcAndRatisThreeKeys() .getSafeModeRule(ECMinDataNodeSafeModeRule.class); assertTrue(ecRule.isEnabled()); + ecRule.setValidateBasedOnReportProcessing(false); GenericTestUtils.waitFor(ecRule::validate, 1000, 60000); - GenericTestUtils.waitFor(() -> !scmSafeModeManager.getInSafeMode(), 1000, - 60000); + GenericTestUtils.waitFor(() -> { + scmSafeModeManager.refreshAndValidate(); + return !scmSafeModeManager.getInSafeMode(); + }, 1000, 60000); } @AfterEach From bdd3f19e7ca62be0caf5f0d97ae7ec21943a3d41 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Fri, 19 Jun 2026 13:09:36 +0530 Subject: [PATCH 08/10] Addressed Rakesh and Ashish comments. --- .../org/apache/hadoop/hdds/scm/ScmUtils.java | 23 +++++++ .../pipeline/BackgroundPipelineCreator.java | 37 +++++----- .../safemode/ECMinDataNodeSafeModeRule.java | 19 ++---- .../TestBackgroundPipelineCreator.java | 6 +- .../TestECMinDataNodeSafeModeRule.java | 48 +++++++++++++ .../scm/safemode/TestSafeModeRuleFactory.java | 68 ++++++++++++++++--- 6 files changed, 154 insertions(+), 47 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java index 21685daebd62..a5465aa993de 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java @@ -45,6 +45,7 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.concurrent.BlockingQueue; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.events.SCMEvents; @@ -54,6 +55,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReport; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; @@ -218,4 +220,25 @@ public static void checkIfCertSignRequestAllowed( } } } + + /** + * Returns default replication config, or null when configured values are + * invalid. Callers can decide whether to fallback or skip their operation. + */ + public static ReplicationConfig getDefaultReplicationConfig( + ConfigurationSource conf, Logger logger, String componentName) { + try { + return ReplicationConfig.getDefault(conf); + } catch (IllegalArgumentException e) { + logger.warn("Ignoring invalid default replication config in {}: " + + "type={}, replication={}.", + componentName, + conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT), + conf.get(OzoneConfigKeys.OZONE_REPLICATION, + OzoneConfigKeys.OZONE_REPLICATION_DEFAULT), + e); + return null; + } + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java index 15f80659aabe..59880dc70e40 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java @@ -44,9 +44,9 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.ScmUtils; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.SCMService; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -236,10 +236,20 @@ private void createPipelines() throws RuntimeException { LOG.debug("BackgroundPipelineCreator createPipelines finished."); } + /** + * Returns replication configs eligible for background pipeline creation. + * + *

If the default replication config is invalid, this returns an empty + * list and skips pipeline creation to avoid guessing from raw config values. + * For EC-default clusters, this only returns RATIS/THREE when + * {@link ScmConfigKeys#OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE} is enabled. + */ @VisibleForTesting List getReplicationConfigs(boolean autoCreateFactorOne) { List list = new ArrayList<>(); - ReplicationConfig defaultReplicationConfig = getDefaultReplicationConfig(); + ReplicationConfig defaultReplicationConfig = ScmUtils + .getDefaultReplicationConfig(conf, LOG, + BackgroundPipelineCreator.class.getSimpleName()); if (defaultReplicationConfig == null) { LOG.warn("Skipping background pipeline creation: default replication " + "config is invalid."); @@ -248,11 +258,11 @@ List getReplicationConfigs(boolean autoCreateFactorOne) { // TODO: #CLUTIL Different replication factor may need to be supported HddsProtos.ReplicationType type = defaultReplicationConfig.getReplicationType(); + if (type == EC && createRatisThreeForEcDefault) { + list.add(ReplicationConfig.fromProtoTypeAndFactor(RATIS, + ReplicationFactor.THREE)); + } if (type == EC) { - if (createRatisThreeForEcDefault) { - list.add(ReplicationConfig.fromProtoTypeAndFactor(RATIS, - ReplicationFactor.THREE)); - } return list; } @@ -274,21 +284,6 @@ List getReplicationConfigs(boolean autoCreateFactorOne) { return list; } - private ReplicationConfig getDefaultReplicationConfig() { - try { - return ReplicationConfig.getDefault(conf); - } catch (IllegalArgumentException e) { - LOG.warn("Ignoring invalid default replication config: type={}, " - + "replication={}.", - conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, - OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT), - conf.get(OzoneConfigKeys.OZONE_REPLICATION, - OzoneConfigKeys.OZONE_REPLICATION_DEFAULT), - e); - return null; - } - } - @Override public void notifyStatusChanged() { serviceLock.lock(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java index af80053c3db3..1add969c5d11 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/ECMinDataNodeSafeModeRule.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.ScmUtils; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.node.NodeStatus; @@ -57,7 +58,9 @@ public ECMinDataNodeSafeModeRule(EventQueue eventQueue, super(safeModeManager, eventQueue); this.nodeManager = nodeManager; - ReplicationConfig defaultConfig = getDefaultReplicationConfig(conf); + ReplicationConfig defaultConfig = ScmUtils.getDefaultReplicationConfig( + conf, SCMSafeModeManager.getLogger(), + ECMinDataNodeSafeModeRule.class.getSimpleName()); if (defaultConfig != null && defaultConfig.getReplicationType() == HddsProtos.ReplicationType.EC) { ECReplicationConfig ecConfig = (ECReplicationConfig) defaultConfig; @@ -142,21 +145,7 @@ synchronized int getRegisteredDns() { return registeredDnSet.size(); } - @VisibleForTesting boolean isEnabled() { return enabled; } - - private static ReplicationConfig getDefaultReplicationConfig( - ConfigurationSource conf) { - try { - return ReplicationConfig.getDefault(conf); - } catch (IllegalArgumentException e) { - SCMSafeModeManager.getLogger().warn( - "Could not parse default replication config for " - + "ECMinDataNodeSafeModeRule; disabling EC rule.", - e); - return null; - } - } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java index 2bbe9fd8c38c..cce0d37f040e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java @@ -100,11 +100,13 @@ public void testRatisDefaultReplicationBehaviorUnchanged() throws Exception { @Test public void testInvalidDefaultReplicationConfigCreatesNoPipelines() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.RATIS.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.RATIS.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "invalid-replication-value"); BackgroundPipelineCreator creator = - new BackgroundPipelineCreator(mock(PipelineManager.class), conf, mock(SCMContext.class), Clock.systemUTC()); + new BackgroundPipelineCreator(mock(PipelineManager.class), conf, + mock(SCMContext.class), Clock.systemUTC()); List configs = creator.getReplicationConfigs(false); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java index c1a61a6e2c49..fe3cc9fb8cd6 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestECMinDataNodeSafeModeRule.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.safemode; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -27,8 +28,10 @@ import java.util.List; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer.NodeRegistrationContainerReport; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.junit.jupiter.api.Test; @@ -84,4 +87,49 @@ public void testEnabledForEcDefaultAndUsesRequiredNodeCount() { assertTrue(rule.validate()); assertFalse(rule.validate()); } + + @Test + public void testProcessCountsAndDeduplicatesRegisteredDnsInReportMode() { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + + NodeManager nodeManager = mock(NodeManager.class); + SCMSafeModeManager safeModeManager = mock(SCMSafeModeManager.class); + when(safeModeManager.getSafeModeMetrics()).thenReturn(mock(SafeModeMetrics.class)); + + ECMinDataNodeSafeModeRule rule = new ECMinDataNodeSafeModeRule( + new EventQueue(), conf, nodeManager, safeModeManager); + + assertTrue(rule.isEnabled()); + assertFalse(rule.validate()); + assertEquals(0, rule.getRegisteredDns()); + + NodeRegistrationContainerReport report1 = createNodeRegistrationReport(); + NodeRegistrationContainerReport report2 = createNodeRegistrationReport(); + NodeRegistrationContainerReport report3 = createNodeRegistrationReport(); + NodeRegistrationContainerReport report4 = createNodeRegistrationReport(); + NodeRegistrationContainerReport report5 = createNodeRegistrationReport(); + + rule.process(report1); + rule.process(report2); + rule.process(report3); + rule.process(report4); + rule.process(report5); + rule.process(report5); + + assertEquals(5, rule.getRegisteredDns()); + assertTrue(rule.validate()); + } + + private static NodeRegistrationContainerReport createNodeRegistrationReport() { + NodeRegistrationContainerReport report = + mock(NodeRegistrationContainerReport.class); + DatanodeDetails dnDetails = mock(DatanodeDetails.class); + DatanodeID dnId = mock(DatanodeID.class); + when(dnDetails.getID()).thenReturn(dnId); + when(report.getDatanodeDetails()).thenReturn(dnDetails); + return report; + } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java index d6f22e969031..968eb79d35b1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java @@ -24,30 +24,27 @@ import java.lang.reflect.Field; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.ContainerManager; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.junit.jupiter.api.Test; class TestSafeModeRuleFactory { @Test public void testIllegalState() { - // If the initialization is already done by different test, we have to reset it. - try { - final Field instance = SafeModeRuleFactory.class.getDeclaredField("instance"); - instance.setAccessible(true); - instance.set(null, null); - } catch (Exception e) { - throw new RuntimeException(); - } + resetInstance(); assertThrows(IllegalStateException.class, SafeModeRuleFactory::getInstance); } @Test public void testLoadedSafeModeRules() { + resetInstance(); SCMSafeModeManager safeModeManager = initializeSafeModeRuleFactory(); final SafeModeRuleFactory factory = SafeModeRuleFactory.getInstance(); factory.addSafeModeManager(safeModeManager); @@ -63,6 +60,7 @@ public void testLoadedSafeModeRules() { @Test public void testLoadedPreCheckRules() { + resetInstance(); SCMSafeModeManager safeModeManager = initializeSafeModeRuleFactory(); final SafeModeRuleFactory factory = SafeModeRuleFactory.getInstance(); factory.addSafeModeManager(safeModeManager); @@ -76,14 +74,66 @@ public void testLoadedPreCheckRules() { } + @Test + public void testRuleCountForEcDefaultWithRatisThreeFlagDisabled() { + resetInstance(); + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + false); + + SCMSafeModeManager safeModeManager = initializeSafeModeRuleFactory(conf); + final SafeModeRuleFactory factory = SafeModeRuleFactory.getInstance(); + factory.addSafeModeManager(safeModeManager); + + assertEquals(4, factory.getSafeModeRules().size(), + "EC default with flag=false should skip RATIS/THREE pipeline rules"); + } + + @Test + public void testRuleCountForEcDefaultWithRatisThreeFlagEnabled() { + resetInstance(); + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + HddsProtos.ReplicationType.EC.name()); + conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + true); + + SCMSafeModeManager safeModeManager = initializeSafeModeRuleFactory(conf); + final SafeModeRuleFactory factory = SafeModeRuleFactory.getInstance(); + factory.addSafeModeManager(safeModeManager); + + assertEquals(6, factory.getSafeModeRules().size(), + "EC default with flag=true should include RATIS/THREE pipeline rules"); + } + private SCMSafeModeManager initializeSafeModeRuleFactory() { + return initializeSafeModeRuleFactory(new OzoneConfiguration()); + } + + private SCMSafeModeManager initializeSafeModeRuleFactory( + OzoneConfiguration conf) { final SCMSafeModeManager safeModeManager = mock(SCMSafeModeManager.class); when(safeModeManager.getSafeModeMetrics()).thenReturn(mock(SafeModeMetrics.class)); - SafeModeRuleFactory.initialize(new OzoneConfiguration(), + SafeModeRuleFactory.initialize(conf, SCMContext.emptyContext(), new EventQueue(), mock( PipelineManager.class), mock(ContainerManager.class), mock(NodeManager.class)); return safeModeManager; } + private static void resetInstance() { + try { + final Field instance = SafeModeRuleFactory.class.getDeclaredField( + "instance"); + instance.setAccessible(true); + instance.set(null, null); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } From bcafe39c7c18fdb94fb0f98e0dfb3a792d9caa62 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Fri, 19 Jun 2026 13:22:43 +0530 Subject: [PATCH 09/10] Fix pmd. --- .../hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java index 4681f00c6d7f..8b63a10a3326 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java @@ -116,12 +116,12 @@ private SCMSafeModeManager initializeSafeModeRuleFactory() { } private SCMSafeModeManager initializeSafeModeRuleFactory( - OzoneConfiguration conf) { + OzoneConfiguration configuration) { final SCMSafeModeManager safeModeManager = mock(SCMSafeModeManager.class); when(safeModeManager.getSafeModeMetrics()).thenReturn(mock(SafeModeMetrics.class)); - OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(HddsConfigKeys.HDDS_SCM_SAFEMODE_RULE_REFRESH_INTERVAL, "0s"); - SafeModeRuleFactory.initialize(conf, + configuration.set(HddsConfigKeys.HDDS_SCM_SAFEMODE_RULE_REFRESH_INTERVAL, + "0s"); + SafeModeRuleFactory.initialize(configuration, SCMContext.emptyContext(), new EventQueue(), mock( PipelineManager.class), mock(ContainerManager.class), mock(NodeManager.class)); From 52976ebf74d5aadf522cdaded71cf077ef75bd4c Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Fri, 19 Jun 2026 23:40:07 +0530 Subject: [PATCH 10/10] Changed ozone.scm.pipeline.creation.ec.ratis.three.enabled to ozone.scm.pipeline.creation.ratis.three --- .../java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +++--- hadoop-hdds/common/src/main/resources/ozone-default.xml | 8 +++++--- .../hdds/scm/pipeline/BackgroundPipelineCreator.java | 6 +++--- .../hadoop/hdds/scm/safemode/SafeModeRuleFactory.java | 6 +++--- .../hdds/scm/pipeline/TestBackgroundPipelineCreator.java | 4 ++-- .../hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java | 4 ++-- .../hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java | 4 ++-- .../scm/safemode/TestSCMSafeModeWithPipelineRules.java | 2 +- 8 files changed, 21 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 12ed791c9be1..93fb3ec19330 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -465,11 +465,11 @@ public final class ScmConfigKeys { * mixed-workload clusters. If false, RATIS/THREE pipeline creation is * skipped for EC-default clusters. */ - public static final String OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE = - "ozone.scm.pipeline.creation.ec.ratis.three.enabled"; + public static final String OZONE_SCM_PIPELINE_CREATE_RATIS_THREE = + "ozone.scm.pipeline.creation.ratis.three"; public static final boolean - OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE_DEFAULT = true; + OZONE_SCM_PIPELINE_CREATE_RATIS_THREE_DEFAULT = true; public static final String OZONE_SCM_BLOCK_DELETION_PER_DN_DISTRIBUTION_FACTOR = "ozone.scm.block.deletion.per.dn.distribution.factor"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 2ac794001d4f..dd33a02b02a7 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1710,12 +1710,14 @@ - ozone.scm.pipeline.creation.ec.ratis.three.enabled + ozone.scm.pipeline.creation.ratis.three true OZONE, SCM, PIPELINE - If enabled, SCM creates RATIS/THREE background pipelines even when the - default replication type is EC. + When true, SCM creates RATIS/THREE pipelines in the background and + requires them during safemode. Applies only when the cluster default + replication type is EC. For RATIS-default clusters this flag has no + effect. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java index 59880dc70e40..97455d56d511 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java @@ -112,8 +112,8 @@ public class BackgroundPipelineCreator implements SCMService { ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); this.createRatisThreeForEcDefault = conf.getBoolean( - ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, - ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE_DEFAULT); + ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, + ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE_DEFAULT); threadName = scmContext.threadNamePrefix() + THREAD_NAME; } @@ -242,7 +242,7 @@ private void createPipelines() throws RuntimeException { *

If the default replication config is invalid, this returns an empty * list and skips pipeline creation to avoid guessing from raw config values. * For EC-default clusters, this only returns RATIS/THREE when - * {@link ScmConfigKeys#OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE} is enabled. + * {@link ScmConfigKeys#OZONE_SCM_PIPELINE_CREATE_RATIS_THREE} is enabled. */ @VisibleForTesting List getReplicationConfigs(boolean autoCreateFactorOne) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java index 1e91aa881974..bb7056d2c3c1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java @@ -109,7 +109,7 @@ private void loadRules(SCMSafeModeManager safeModeManager) { "RATIS/THREE pipeline safemode rules are disabled because " + "{} is false for an EC-default cluster or the default " + "replication config is invalid.", - ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE); + ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE); } } @@ -139,8 +139,8 @@ private boolean shouldEnableRatisThreePipelineRules() { return true; } - return config.getBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, - ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE_DEFAULT); + return config.getBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, + ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE_DEFAULT); } public static synchronized SafeModeRuleFactory getInstance() { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java index cce0d37f040e..e297266cc285 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineCreator.java @@ -45,7 +45,7 @@ public void testEcDefaultReplicationWithoutRatisThreeFlagCreatesNoPipelines() conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + conf.setBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, false); BackgroundPipelineCreator creator = new BackgroundPipelineCreator( @@ -63,7 +63,7 @@ public void testEcDefaultReplicationWithRatisThreeFlag() throws Exception { conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + conf.setBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, true); BackgroundPipelineCreator creator = new BackgroundPipelineCreator( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index 7e8c76592020..cf262873eb0e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -831,7 +831,7 @@ public void testEcDefaultDisablesHealthyPipelineRuleWhenRatisThreeDisabled() { conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + conf.setBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, false); MockNodeManager mockNodeManager = new MockNodeManager(true, 5); @@ -870,7 +870,7 @@ public void testEcDefaultKeepsHealthyPipelineRuleWhenRatisThreeEnabled() { conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + conf.setBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, true); MockNodeManager mockNodeManager = new MockNodeManager(true, 5); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java index 8b63a10a3326..3f0b8f415f2b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSafeModeRuleFactory.java @@ -82,7 +82,7 @@ public void testRuleCountForEcDefaultWithRatisThreeFlagDisabled() { conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + conf.setBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, false); SCMSafeModeManager safeModeManager = initializeSafeModeRuleFactory(conf); @@ -100,7 +100,7 @@ public void testRuleCountForEcDefaultWithRatisThreeFlagEnabled() { conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + conf.setBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, true); SCMSafeModeManager safeModeManager = initializeSafeModeRuleFactory(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java index 27dd1fad089b..4a74d94a67b6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java @@ -175,7 +175,7 @@ void testSafeModeExitAfterScmRestartWithMixedEcAndRatisThreeKeys() conf.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, HddsProtos.ReplicationType.EC.name()); conf.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024k"); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_EC_PIPELINE_CREATE_RATIS_THREE, + conf.setBoolean(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATE_RATIS_THREE, true); cluster = MiniOzoneCluster.newBuilder(conf)