Skip to content

Commit 1486d88

Browse files
committed
Revert "Fix IoTConsensus batch accumulation by byte size (#18560)"
This reverts commit c41838e.
1 parent d61ef99 commit 1486d88

3 files changed

Lines changed: 7 additions & 83 deletions

File tree

‎iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/Batch.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,6 @@ public void addTLogEntry(TLogEntry entry, boolean containsUserData) {
6969
}
7070

7171
public boolean canAccumulate() {
72-
return canAccumulate(config, logEntries.size(), memorySize);
73-
}
74-
75-
static boolean canAccumulate(IoTConsensusConfig config, int logEntriesSize, long memorySize) {
7672
// When reading entries from the WAL, the memory size is calculated based on the serialized
7773
// size, which can be significantly smaller than the actual size.
7874
// Thus, we add a multiplier to sender's memory size to estimate the receiver's memory cost.
@@ -81,7 +77,7 @@ static boolean canAccumulate(IoTConsensusConfig config, int logEntriesSize, long
8177
long senderMemSize = LogDispatcher.getSenderMemSizeSum().get();
8278
double multiplier = senderMemSize > 0 ? (double) receiverMemSize / senderMemSize : 1.0;
8379
multiplier = Math.max(multiplier, 1.0);
84-
return logEntriesSize < config.getReplication().getMaxLogEntriesNumPerBatch()
80+
return logEntries.size() < config.getReplication().getMaxLogEntriesNumPerBatch()
8581
&& ((long) (memorySize * multiplier)) < config.getReplication().getMaxSizePerBatch();
8682
}
8783

‎iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -432,16 +432,12 @@ void waitForBatchAccumulation(long waitingTimeInMs) throws InterruptedException
432432
}
433433

434434
final long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(waitingTimeInMs);
435-
final IoTConsensusConfig currentConfig = config;
436-
int accumulatedEntries = bufferedEntries.size();
437-
long accumulatedMemorySize =
438-
bufferedEntries.stream().mapToLong(IndexedConsensusRequest::getMemorySize).sum();
439-
440-
// Keep collecting while the batch is below both its entry and memory limits. A plain sleep,
441-
// or checking only the entry limit, makes the dispatcher wait for the full accumulation
442-
// interval after a batch has already reached its memory limit. This unnecessarily throttles
443-
// IoTConsensus when each request contains a large tablet.
444-
while (Batch.canAccumulate(currentConfig, accumulatedEntries, accumulatedMemorySize)) {
435+
final int maxLogEntriesNumPerBatch = config.getReplication().getMaxLogEntriesNumPerBatch();
436+
437+
// Keep collecting while the batch is below its entry limit. A plain sleep makes the
438+
// dispatcher wait for the full accumulation interval even when the batch becomes full
439+
// immediately, which unnecessarily throttles IoTConsensus under sustained write load.
440+
while (bufferedEntries.size() < maxLogEntriesNumPerBatch) {
445441
final long remainingNanos = deadlineNanos - System.nanoTime();
446442
if (remainingNanos <= 0) {
447443
return;
@@ -453,8 +449,6 @@ void waitForBatchAccumulation(long waitingTimeInMs) throws InterruptedException
453449
return;
454450
}
455451
bufferedEntries.add(request);
456-
accumulatedEntries++;
457-
accumulatedMemorySize += request.getMemorySize();
458452
}
459453
}
460454

‎iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcherTest.java‎

Lines changed: 0 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -177,72 +177,6 @@ public void sendBatchAsync(Batch sentBatch, DispatchLogHandler handler) {
177177
}
178178
}
179179

180-
@Test
181-
public void testBatchAccumulationStopsWhenMemoryLimitIsReached() throws Exception {
182-
final Peer localPeer = createPeer(1, 6697);
183-
final Peer remotePeer = createPeer(2, 6698);
184-
final IoTConsensusConfig config =
185-
IoTConsensusConfig.newBuilder()
186-
.setReplication(
187-
IoTConsensusConfig.Replication.newBuilder()
188-
.setMaxLogEntriesNumPerBatch(1024)
189-
.setMaxSizePerBatch(1)
190-
.setMaxWaitingTimeForAccumulatingBatchInMs(10_000)
191-
.build())
192-
.build();
193-
final ScheduledExecutorService backgroundTaskService =
194-
Executors.newSingleThreadScheduledExecutor();
195-
final ExecutorService executorService = Executors.newSingleThreadExecutor();
196-
LogDispatcher.LogDispatcherThread dispatcherThread = null;
197-
Future<?> dispatcherFuture = null;
198-
try {
199-
final IoTConsensusServerImpl server =
200-
createServer(
201-
localPeer, Collections.singletonList(localPeer), config, backgroundTaskService);
202-
final CountDownLatch batchSent = new CountDownLatch(1);
203-
final AtomicInteger getBatchInvocations = new AtomicInteger();
204-
dispatcherThread =
205-
server.getLogDispatcher().new LogDispatcherThread(remotePeer, config, 0) {
206-
@Override
207-
public Batch getBatch() {
208-
return getBatchInvocations.getAndIncrement() == 0
209-
? new Batch(config)
210-
: createBatch(config, 1);
211-
}
212-
213-
@Override
214-
public void sendBatchAsync(Batch sentBatch, DispatchLogHandler handler) {
215-
assertEquals(1, getPendingEntriesSize());
216-
assertEquals(1, getBufferRequestSize());
217-
batchSent.countDown();
218-
Thread.currentThread().interrupt();
219-
}
220-
};
221-
final IndexedConsensusRequest firstRequest =
222-
new IndexedConsensusRequest(1, Collections.singletonList(new TestEntry(1, localPeer)));
223-
firstRequest.buildSerializedRequests();
224-
final IndexedConsensusRequest secondRequest =
225-
new IndexedConsensusRequest(2, Collections.singletonList(new TestEntry(2, localPeer)));
226-
secondRequest.buildSerializedRequests();
227-
assertTrue(dispatcherThread.offer(firstRequest));
228-
assertTrue(dispatcherThread.offer(secondRequest));
229-
230-
dispatcherFuture = executorService.submit(dispatcherThread);
231-
assertTrue(batchSent.await(2, TimeUnit.SECONDS));
232-
dispatcherFuture.get(2, TimeUnit.SECONDS);
233-
} finally {
234-
if (dispatcherFuture != null) {
235-
dispatcherFuture.cancel(true);
236-
}
237-
executorService.shutdownNow();
238-
executorService.awaitTermination(5, TimeUnit.SECONDS);
239-
if (dispatcherThread != null) {
240-
dispatcherThread.stop();
241-
}
242-
backgroundTaskService.shutdownNow();
243-
}
244-
}
245-
246180
@Test
247181
public void testReloadConfigUpdatesExistingDispatcherPipeline() throws Exception {
248182
final Peer localPeer = createPeer(1, 6677);

0 commit comments

Comments
 (0)