Skip to content

[fix][test] Replace fixed sleeps with Awaitility polls in sink tests #86

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
david-streamlio merged 2 commits into from
Jul 10, 2026
Merged

[fix][test] Replace fixed sleeps with Awaitility polls in sink tests #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4b91199
[fix][test] Replace fixed sleeps with Awaitility polls in sink tests
david-streamlio Jul 10, 2026
14ffdd5
Potential fix for pull request finding
david-streamlio Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions hbase/src/test/java/org/apache/pulsar/io/hbase/sink/HbaseGenericRecordSinkTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.hbase.client.Get;
Expand All @@ -46,6 +47,7 @@
import org.apache.pulsar.functions.source.PulsarSourceConfig;
import org.apache.pulsar.io.core.SinkContext;
import org.apache.pulsar.io.hbase.TableUtils;
import org.awaitility.Awaitility;
import org.mockito.Mock;
import org.testng.Assert;
import org.testng.annotations.Test;
Expand Down Expand Up @@ -155,12 +157,13 @@ public void testOpenAndWriteSink() throws Exception {
// write should success.
sink.write(record);
log.info("executed write");
// sleep to wait backend flush complete
Thread.sleep(500);

// value has been written to hbase table, read it out and verify.
// The sink flushes on a background executor; poll for the row rather than racing it.
Table table = TableUtils.getTable(map);
Get scan = new Get(Bytes.toBytes(obj.getRowKey()));
Awaitility.await().atMost(30, TimeUnit.SECONDS)
.pollInterval(100, TimeUnit.MILLISECONDS)
.until(() -> !table.get(scan).isEmpty());

Result result = table.get(scan);
byte[] byteName = result.getValue(Bytes.toBytes(familyName), Bytes.toBytes(name));
byte[] byteAddress = result.getValue(Bytes.toBytes(familyName), Bytes.toBytes(address));
Expand Down
12 changes: 8 additions & 4 deletions hdfs3/src/test/java/org/apache/pulsar/io/hdfs3/sink/seq/HdfsSequentialSinkTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@
package org.apache.pulsar.io.hdfs3.sink.seq;

import static org.mockito.Mockito.times;
import java.util.concurrent.TimeUnit;
import static org.mockito.Mockito.verify;
import static org.testng.Assert.assertNotNull;
import org.apache.pulsar.io.hdfs3.sink.AbstractHdfsSinkTest;
import org.awaitility.Awaitility;
import org.testng.SkipException;
import org.testng.annotations.Test;

Expand All @@ -42,8 +44,9 @@ public final void write100Test() throws Exception {
assertNotNull(sink);
send(100);

Thread.sleep(2000);
verify(mockRecord, times(100)).ack();
// The sink syncs on a background interval; poll rather than racing it.
Awaitility.await().atMost(60, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(100)).ack());
sink.close();
}

Expand All @@ -57,8 +60,9 @@ public final void write5000Test() throws Exception {
assertNotNull(sink);
send(5000);

Thread.sleep(2000);
verify(mockRecord, times(5000)).ack();
// The sink syncs on a background interval; poll rather than racing it.
Awaitility.await().atMost(60, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(5000)).ack());
sink.close();
}

Expand Down
12 changes: 8 additions & 4 deletions hdfs3/src/test/java/org/apache/pulsar/io/hdfs3/sink/seq/HdfsTextSinkTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.testng.Assert.assertNotNull;
import java.util.concurrent.TimeUnit;
import org.apache.pulsar.io.hdfs3.sink.AbstractHdfsSinkTest;
import org.awaitility.Awaitility;
import org.testng.SkipException;
import org.testng.annotations.Test;

Expand All @@ -43,8 +45,9 @@ public final void write100Test() throws Exception {
assertNotNull(mockRecord);
send(100);

Thread.sleep(2000);
verify(mockRecord, times(100)).ack();
// The sink syncs on a background interval; poll rather than racing it.
Awaitility.await().atMost(60, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(100)).ack());
sink.close();
}

Expand All @@ -59,8 +62,9 @@ public final void write5000Test() throws Exception {
assertNotNull(mockRecord);
send(5000);

Thread.sleep(2000);
verify(mockRecord, times(5000)).ack();
// The sink syncs on a background interval; poll rather than racing it.
Awaitility.await().atMost(60, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(5000)).ack());
sink.close();
}

Expand Down
8 changes: 5 additions & 3 deletions influxdb/src/test/java/org/apache/pulsar/io/influxdb/v1/InfluxDBGenericRecordSinkTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import static org.mockito.Mockito.when;
import com.google.common.collect.Maps;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.client.api.Message;
Expand All @@ -39,6 +40,7 @@
import org.apache.pulsar.functions.api.Record;
import org.apache.pulsar.functions.source.PulsarRecord;
import org.apache.pulsar.io.core.SinkContext;
import org.awaitility.Awaitility;
import org.influxdb.InfluxDB;
import org.influxdb.dto.BatchPoints;
import org.mockito.Mock;
Expand Down Expand Up @@ -150,8 +152,8 @@ public void testOpenAndWrite() throws Exception {

influxSink.write(record);

Thread.sleep(1000);

verify(influxDB, times(1)).write(any(BatchPoints.class));
// BatchSink flushes on a background executor; poll rather than racing it.
Awaitility.await().atMost(30, TimeUnit.SECONDS)
.untilAsserted(() -> verify(influxDB, times(1)).write(any(BatchPoints.class)));
}
}
36 changes: 20 additions & 16 deletions mongo/src/test/java/org/apache/pulsar/io/mongodb/MongoSinkTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.pulsar.functions.api.Record;
import org.apache.pulsar.io.core.SinkContext;
import org.awaitility.Awaitility;
import org.bson.BsonDocument;
import org.mockito.Mock;
import org.reactivestreams.Publisher;
Expand Down Expand Up @@ -149,9 +151,9 @@ public void testWriteNullMessage() throws Exception {
sink.open(map, mockSinkContext);
sink.write(mockRecord);

Thread.sleep(1000);

verify(mockRecord, times(1)).fail();
// The sink flushes on a background executor; poll rather than racing it.
Awaitility.await().atMost(30, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(1)).fail());
}

@Test
Expand All @@ -161,9 +163,9 @@ public void testWriteGoodMessage() throws Exception {
sink.open(map, mockSinkContext);
sink.write(mockRecord);

Thread.sleep(1000);

verify(mockRecord, times(1)).ack();
// The sink flushes on a background executor; poll rather than racing it.
Awaitility.await().atMost(30, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(1)).ack());
}

@Test
Expand All @@ -175,10 +177,12 @@ public void testWriteMultipleMessages() throws Exception {
sink.write(mockRecord);
sink.write(mockRecord);

Thread.sleep(1000);

verify(mockRecord, times(2)).ack();
verify(mockRecord, times(1)).fail();
// The sink flushes on a background executor; poll rather than racing it.
Awaitility.await().atMost(30, TimeUnit.SECONDS)
.untilAsserted(() -> {
verify(mockRecord, times(2)).ack();
verify(mockRecord, times(1)).fail();
});
}

@Test
Expand All @@ -188,9 +192,9 @@ public void testWriteWithError() throws Exception {
sink.open(map, mockSinkContext);
sink.write(mockRecord);

Thread.sleep(1000);

verify(mockRecord, times(1)).fail();
// The sink flushes on a background executor; poll rather than racing it.
Awaitility.await().atMost(30, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(1)).fail());
}

@Test
Expand All @@ -200,8 +204,8 @@ public void testWriteBadMessage() throws Exception {
sink.open(map, mockSinkContext);
sink.write(mockRecord);

Thread.sleep(1000);

verify(mockRecord, times(1)).fail();
// The sink flushes on a background executor; poll rather than racing it.
Awaitility.await().atMost(30, TimeUnit.SECONDS)
.untilAsserted(() -> verify(mockRecord, times(1)).fail());
}
}
Loading