Replace Thread.sleep with OpenSearchTestCase.waitUntil in tests (opensearch-project#2041)

Replace all remaining Thread.sleep calls in integration tests with
OpenSearchTestCase.waitUntil for more reliable test execution.

Resolves opensearch-project#1028

Signed-off-by: Manaswini Ragamouni <ragamanu@amazon.com>
Co-authored-by: Manaswini Ragamouni <ragamanu@amazon.com>

Author: manaswini1920

alerting/src/test/kotlin/org/opensearch/alerting/DocLeveFanOutIT.kt

@@ -12,9 +12,11 @@ import org.opensearch.commons.alerting.model.DocLevelMonitorInput
 import org.opensearch.commons.alerting.model.DocLevelQuery
 import org.opensearch.commons.alerting.model.action.ActionExecutionPolicy
 import org.opensearch.commons.alerting.model.action.PerExecutionActionScope
+import org.opensearch.test.OpenSearchTestCase
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 import java.time.temporal.ChronoUnit.MILLIS
+import java.util.concurrent.TimeUnit
 
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE)
 class DocLeveFanOutIT : AlertingRestTestCase() {
@@ -64,7 +66,9 @@ class DocLeveFanOutIT : AlertingRestTestCase() {
         assertEquals(findingsSize1, 2)
         adminClient().updateSettings(AlertingSettings.DOC_LEVEL_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1))
         executeMonitor(monitor.id)
-        Thread.sleep(1000)
+        OpenSearchTestCase.waitUntil({
+            return@waitUntil true
+        }, 2, TimeUnit.SECONDS)
         adminClient().updateSettings(AlertingSettings.DOC_LEVEL_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueMinutes(4))
         indexDoc(testIndex, "3", testDoc)
         indexDoc(testIndex, "4", testDoc)

alerting/src/test/kotlin/org/opensearch/alerting/DocumentMonitorRunnerIT.kt

@@ -266,7 +266,9 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
         updateSettings =
             adminClient().updateSettings(AlertingSettings.DOC_LEVEL_MONITOR_FANOUT_MAX_DURATION.key, TimeValue.timeValueMinutes(4))
         logger.info(updateSettings)
-        Thread.sleep(1000)
+        OpenSearchTestCase.waitUntil({
+            return@waitUntil true
+        }, 2, TimeUnit.SECONDS)
         response = executeMonitor(monitor.id)
         output = entityAsMap(response)
         assertEquals(monitor.name, output["monitor_name"])
@@ -597,10 +599,14 @@ class DocumentMonitorRunnerIT : AlertingRestTestCase() {
 
         indexDoc(testIndex, "1", testDoc)
         indexDoc(testIndex, "5", testDoc)
-        Thread.sleep(240000)
 
         val inputMap = HashMap<String, Any>()
         inputMap["searchString"] = monitor.name
+        OpenSearchTestCase.waitUntil({
+            val responseMap = getAlerts(inputMap).asMap()
+            val alerts = (responseMap["alerts"] as ArrayList<Map<String, Any>>)
+            return@waitUntil alerts.isNotEmpty()
+        }, 5, TimeUnit.MINUTES)
 
         val responseMap = getAlerts(inputMap).asMap()
         val alerts = (responseMap["alerts"] as ArrayList<Map<String, Any>>)

alerting/src/test/kotlin/org/opensearch/alerting/MonitorRunnerServiceIT.kt

@@ -1267,7 +1267,9 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() {
 
     fun `test execute bucket-level monitor with alias optimization - indices skipped from query`() {
         val skipIndex = createTestIndex("to_skip_index")
-        Thread.sleep(10000)
+        OpenSearchTestCase.waitUntil({
+            return@waitUntil true
+        }, 10, TimeUnit.SECONDS)
         val previousIndex = createTestIndex("to_include_index")
         insertSampleTimeSerializedDataWithTime(
             previousIndex,
@@ -1287,7 +1289,9 @@ class MonitorRunnerServiceIT : AlertingRestTestCase() {
             ),
             ZonedDateTime.now().truncatedTo(ChronoUnit.MILLIS).plusSeconds(10)
         )
-        Thread.sleep(10000)
+        OpenSearchTestCase.waitUntil({
+            return@waitUntil true
+        }, 10, TimeUnit.SECONDS)
         val indexMapping = """
                 "properties" : {
                   "test_strict_date_time" : { "type" : "date", "format" : "strict_date_time" },

alerting/src/test/kotlin/org/opensearch/alerting/resthandler/WorkflowRestApiIT.kt

@@ -1226,7 +1226,11 @@ class WorkflowRestApiIT : AlertingRestTestCase() {
 
         indexDoc(testIndex, "1", testDoc)
         indexDoc(testIndex, "5", testDoc)
-        Thread.sleep(240000)
+
+        OpenSearchTestCase.waitUntil({
+            val alerts = searchAlerts(monitor)
+            return@waitUntil alerts.isNotEmpty()
+        }, 5, TimeUnit.MINUTES)
 
         val alerts = searchAlerts(monitor)
         alerts.forEach {