From c5d29a9f60b536a10a585641e1a2632be1931288 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Thu, 29 Aug 2024 10:39:05 -0700 Subject: [PATCH] test fixes Signed-off-by: Marc Handalian --- .../SearchOnlyReplicaFeatureFlagIT.java | 13 ---- .../indices/settings/SearchOnlyReplicaIT.java | 60 +++++++++++++------ 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java index 4f6de1fe87baf..e5a05c04fa7ee 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaFeatureFlagIT.java @@ -17,7 +17,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; -import static org.opensearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1) public class SearchOnlyReplicaFeatureFlagIT extends OpenSearchIntegTestCase { @@ -32,18 +31,8 @@ protected Settings featureFlagSettings() { .build(); } - @Override - public Settings indexSettings() { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) - .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .build(); - } - public void testCreateFeatureFlagDisabled() { Settings settings = Settings.builder().put(indexSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, false).build(); - SettingsException settingsException = expectThrows(SettingsException.class, () -> createIndex(TEST_INDEX, settings)); assertTrue(settingsException.getMessage().contains("unknown setting")); } @@ -51,8 +40,6 @@ public void testCreateFeatureFlagDisabled() { public void testUpdateFeatureFlagDisabled() { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "0ms") // so that after we punt a node we can immediately try to - // reallocate after node left. .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java index 05f6e4ef4caab..5fc8e30ed2c7a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/SearchOnlyReplicaIT.java @@ -8,6 +8,7 @@ package org.opensearch.indices.settings; +import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; @@ -24,6 +25,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class SearchOnlyReplicaIT extends OpenSearchIntegTestCase { @@ -79,9 +81,9 @@ public void testUpdateDocRepFails() { assertEquals(expectedFailureMessage, illegalArgumentException.getMessage()); } - public void testSearchReplicasAreNotPrimaryEligible() throws IOException { - int numSearchReplicas = randomIntBetween(0, 3); - int numWriterReplicas = randomIntBetween(0, 3); + public void testFailoverWithSearchReplica_WithWriterReplicas() throws IOException { + int numSearchReplicas = 1; + int numWriterReplicas = 1; internalCluster().startClusterManagerOnlyNode(); String primaryNodeName = internalCluster().startDataOnlyNode(); createIndex( @@ -93,9 +95,8 @@ public void testSearchReplicasAreNotPrimaryEligible() throws IOException { .build() ); ensureYellow(TEST_INDEX); - for (int i = 0; i < numSearchReplicas + numWriterReplicas; i++) { - internalCluster().startDataOnlyNode(); - } + // add 2 nodes for the replicas + internalCluster().startDataOnlyNodes(2); ensureGreen(TEST_INDEX); // assert shards are on separate nodes & all active @@ -103,19 +104,42 @@ public void testSearchReplicasAreNotPrimaryEligible() throws IOException { // stop the primary and ensure search shard is not promoted: internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName)); - ensureRed(TEST_INDEX); + ensureYellowAndNoInitializingShards(TEST_INDEX); + + assertActiveShardCounts(numSearchReplicas, 0); // 1 repl is inactive that was promoted to primary + // add back a node + internalCluster().startDataOnlyNode(); + ensureGreen(TEST_INDEX); + + } + + public void testFailoverWithSearchReplica_WithoutWriterReplicas() throws IOException { + int numSearchReplicas = 1; + int numWriterReplicas = 0; + internalCluster().startClusterManagerOnlyNode(); + String primaryNodeName = internalCluster().startDataOnlyNode(); + createIndex( + TEST_INDEX, + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numWriterReplicas) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, numSearchReplicas) + .build() + ); + ensureYellow(TEST_INDEX); + client().prepareIndex(TEST_INDEX).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + // start a node for our search replica + String replica = internalCluster().startDataOnlyNode(); + ensureGreen(TEST_INDEX); + assertActiveSearchShards(numSearchReplicas); + assertHitCount(client(replica).prepareSearch(TEST_INDEX).setSize(0).setPreference("_only_local").get(), 1); - if (numWriterReplicas > 0) { - assertActiveShardCounts(numSearchReplicas, numWriterReplicas - 1); // 1 repl is inactive that was promoted to primary - // add back a node - internalCluster().startDataOnlyNode(); - ensureGreen(TEST_INDEX); - } else { - // index falls red and does not recover - // Without any writer replica with n2n replication this is an unrecoverable scenario and snapshot restore is required. - ensureRed(TEST_INDEX); - assertActiveSearchShards(numSearchReplicas); - } + // stop the primary and ensure search shard is not promoted: + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName)); + ensureRed(TEST_INDEX); + assertActiveSearchShards(numSearchReplicas); + // while red our search shard is still searchable + assertHitCount(client(replica).prepareSearch(TEST_INDEX).setSize(0).setPreference("_only_local").get(), 1); } public void testSearchReplicaScaling() {