From aabb7ca16ef73e12f055d59d563bceafd061748c Mon Sep 17 00:00:00 2001 From: void-ptr974 Date: Sat, 11 Apr 2026 14:53:08 +0800 Subject: [PATCH] [FIX] Fix flaky BookieAutoRecoveryTest#testOpenLedgers timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the killed bookie happens to be the Auditor leader, the test must wait for the ZK session timeout (default 10s) before the ephemeral node disappears, then wait for a new Auditor leader election, metadata scan, and underreplicated ledger publishing. In resource-constrained CI environments, this chain can exceed the 60-second await timeout. Two changes: - setZkTimeout(4000) — reduces ZK session timeout so the ephemeral node disappears faster - await 60s → 90s — provides more headroom for slow CI environments --- .../replication/BookieAutoRecoveryTest.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java index 1367208c302..ed9ca4e716d 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java @@ -88,6 +88,9 @@ public BookieAutoRecoveryTest() throws IOException, KeeperException, "org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory"); baseConf.setOpenLedgerRereplicationGracePeriod(openLedgerRereplicationGracePeriod); baseConf.setRwRereplicateBackoffMs(500); + // Reduce ZK session timeout so killed bookie's ephemeral node disappears faster, + // speeding up Auditor leader re-election when the killed bookie was the leader. + baseConf.setZkTimeout(4000); baseClientConf.setLedgerManagerFactoryClassName( "org.apache.bookkeeper.meta.HierarchicalLedgerManagerFactory"); this.digestType = DigestType.MAC; @@ -168,7 +171,7 @@ public void testOpenLedgers() throws Exception { // waiting to publish urLedger znode by Auditor assertTrue("Ledger should be marked as underreplicated", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); latch = new CountDownLatch(1); LOG.info("Watching on urLedgerPath:" + urLedgerZNode + " to know the status of rereplication process"); @@ -186,7 +189,7 @@ public void testOpenLedgers() throws Exception { + replicaToKillAddr); } assertTrue("Replication should complete", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); // grace period to update the urledger metadata in zookeeper LOG.info("Waiting to update the urledger metadata in zookeeper"); @@ -223,7 +226,7 @@ public void testClosedLedgers() throws Exception { // waiting to publish urLedger znode by Auditor assertTrue("Ledgers should be marked as underreplicated", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); // Again watching the urLedger znode to know the replication status latch = new CountDownLatch(listOfLedgerHandle.size()); @@ -248,7 +251,7 @@ public void testClosedLedgers() throws Exception { // waiting to finish replication assertTrue("Replication should complete", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); // grace period to update the urledger metadata in zookeeper LOG.info("Waiting to update the urledger metadata in zookeeper"); @@ -296,7 +299,7 @@ public void testStopWhileReplicationInProgress() throws Exception { // waiting to publish urLedger znode by Auditor assertTrue("Ledgers should be marked as underreplicated", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); // Again watching the urLedger znode to know the replication status latch = new CountDownLatch(listOfLedgerHandle.size()); @@ -332,7 +335,7 @@ public void testStopWhileReplicationInProgress() throws Exception { LOG.info("Waiting to finish rereplication processes"); assertTrue("Replication should complete after restart", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); // grace period to update the urledger metadata in zookeeper LOG.info("Waiting to update the urledger metadata in zookeeper"); @@ -369,7 +372,7 @@ public void testNoSuchLedgerExists() throws Exception { killBookie(replicaToKillAddr); // waiting to publish urLedger znode by Auditor assertTrue("Ledgers should be marked as underreplicated", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); latch = new CountDownLatch(listOfLedgerHandle.size()); for (LedgerHandle lh : listOfLedgerHandle) { @@ -385,7 +388,7 @@ public void testNoSuchLedgerExists() throws Exception { // waiting to delete published urledgers, since it doesn't exists assertTrue("UrLedgers should be cleaned up after deletion", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); for (LedgerHandle lh : listOfLedgerHandle) { assertNull("UrLedger still exists after rereplication", @@ -491,7 +494,7 @@ public void testLedgerMetadataContainsIpAddressAsBookieID() // waiting to publish urLedger znode by Auditor assertTrue("Ledger should be marked as underreplicated", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); latch = new CountDownLatch(1); LOG.info("Watching on urLedgerPath:" + urLedgerZNode + " to know the status of rereplication process"); @@ -512,7 +515,7 @@ public void testLedgerMetadataContainsIpAddressAsBookieID() + replicaToKillAddr); } assertTrue("Replication should complete", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); // grace period to update the urledger metadata in zookeeper LOG.info("Waiting to update the urledger metadata in zookeeper"); @@ -570,7 +573,7 @@ public void testLedgerMetadataContainsHostNameAsBookieID() // waiting to publish urLedger znode by Auditor assertTrue("Ledger should be marked as underreplicated", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); latch = new CountDownLatch(1); LOG.info("Watching on urLedgerPath:" + urLedgerZNode + " to know the status of rereplication process"); @@ -593,7 +596,7 @@ public void testLedgerMetadataContainsHostNameAsBookieID() + replicaToKillAddr); } assertTrue("Replication should complete", - latch.await(60, TimeUnit.SECONDS)); + latch.await(90, TimeUnit.SECONDS)); // grace period to update the urledger metadata in zookeeper LOG.info("Waiting to update the urledger metadata in zookeeper");