Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,16 @@
import io.axoniq.axonserver.connector.FlowControl;
import io.axoniq.axonserver.connector.ReplyChannel;
import io.axoniq.axonserver.grpc.query.QueryResponse;
import io.axoniq.framework.messaging.distributed.CallbackHandOff;
import org.axonframework.common.annotation.Internal;
import org.axonframework.conversion.Converter;
import org.axonframework.messaging.core.MessageStream;
import org.axonframework.messaging.queryhandling.QueryResponseMessage;
import org.jspecify.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.invoke.MethodHandles;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;

Expand All @@ -32,6 +36,8 @@
@Internal
class FlowControlledResponseSender implements FlowControl {

private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

private final String clientId;
private final String queryIdentifier;
private final MessageStream<QueryResponseMessage> upstream;
Expand Down Expand Up @@ -73,7 +79,8 @@ public void request(long requested) {
if (requests.getAndUpdate(current ->
current > Long.MAX_VALUE - requested ? Long.MAX_VALUE :
current + requested) == 0) {
upstream.setCallback(this::responseSendingLoop);
// The stream calls back on the thread of whoever produced a response, which may not read from it, or block
upstream.setCallback(CallbackHandOff.handOff(this::responseSendingLoop));
}
responseSendingLoop();
}
Expand Down Expand Up @@ -123,11 +130,39 @@ private void sendResponses() {
}
);
}
} catch (RuntimeException e) {
fail(e);
} finally {
sendingGate.set(false);
}
}

/*
* Ends the query with the given failure, as nothing reads the stream after this, and nobody else would tell
* the client: it would wait for the query to time out.
*/
private void fail(RuntimeException failure) {
if (closed.getAndSet(true)) {
// The terminal response was already being sent when this failed; there is nothing left to tell the client
logger.warn("Failed to complete the response to query [{}].", queryIdentifier, failure);
return;
}

upstream.close();

try {
downstream.sendLast(buildErrorResponse(clientId, queryIdentifier, failure, converter));
} catch (RuntimeException e) {
// Most likely the failure's details could not be converted: retry without them
try {
downstream.sendLast(buildErrorResponse(clientId, queryIdentifier, failure, null));
} catch (RuntimeException ex) {
ex.addSuppressed(failure);
logger.warn("Failed to report a failure to the client for query [{}].", queryIdentifier, ex);
}
}
}

@Override
public void cancel() {
upstream.close();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,25 @@
import org.axonframework.messaging.queryhandling.QueryResponseMessage;
import org.jspecify.annotations.NonNull;
import org.junit.jupiter.api.*;
import org.mockito.ArgumentCaptor;

import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;

import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.mockito.Mockito.*;

class FlowControlledResponseSenderTest {

public static final byte[] PAYLOAD = "test".getBytes(StandardCharsets.UTF_8);

// What the sender does when its stream signals happens on a thread of its own, which the tests wait for
private static final long AWAIT_MILLIS = 5000;
private QueueMessageStream<QueryResponseMessage> upstream;
private ReplyChannel<QueryResponse> stubDownstream;
private FlowControlledResponseSender testSubject;
Expand All @@ -56,8 +62,8 @@ void shouldSendAllAlreadyAvailableMessagesUntilCompletionOnUnlimitedPermits() {
}
upstream.seal();
testSubject.request(Long.MAX_VALUE);
verify(stubDownstream, timeout(AWAIT_MILLIS)).complete();
verify(stubDownstream, times(10)).send(any());
verify(stubDownstream).complete();
}

@Test
Expand All @@ -67,8 +73,8 @@ void shouldSendMessagesUntilCompletionOnUnlimitedPermitsWhenMessagesBecomeAvaila
upstream.offer(newMessage(), Context.empty());
}
upstream.seal();
verify(stubDownstream, timeout(AWAIT_MILLIS)).complete();
verify(stubDownstream, times(10)).send(any());
verify(stubDownstream).complete();
}

@Test
Expand All @@ -79,8 +85,8 @@ void shouldNotOverflowOnMoreThanMaxLongRequests() {
upstream.offer(newMessage(), Context.empty());
}
upstream.seal();
verify(stubDownstream, timeout(AWAIT_MILLIS)).complete();
verify(stubDownstream, times(10)).send(any());
verify(stubDownstream).complete();
}

@Test
Expand All @@ -91,9 +97,9 @@ void shouldHonorTheFlowRequestsWhenEnoughMessagesAreAvailable() {
upstream.seal();
for (int i = 0; i < 10; i++) {
testSubject.request(1);
verify(stubDownstream, times(i + 1)).send(any());
verify(stubDownstream, timeout(AWAIT_MILLIS).times(i + 1)).send(any());
}
verify(stubDownstream).complete();
verify(stubDownstream, timeout(AWAIT_MILLIS)).complete();
}

@Test
Expand All @@ -103,10 +109,10 @@ void shouldTriggerCloseWhenUpstreamCompletesAfterSendingAllRequestedMessages() {
}
for (int i = 0; i < 10; i++) {
testSubject.request(1);
verify(stubDownstream, times(i + 1)).send(any());
verify(stubDownstream, timeout(AWAIT_MILLIS).times(i + 1)).send(any());
}
upstream.seal();
verify(stubDownstream).complete();
verify(stubDownstream, timeout(AWAIT_MILLIS)).complete();
}

@Test
Expand All @@ -115,22 +121,22 @@ void shouldOnlySendMessagesWhenRequested() {
upstream.offer(newMessage(), Context.empty());

testSubject.request(1);
verify(stubDownstream, times(1)).send(any());
verify(stubDownstream, timeout(AWAIT_MILLIS).times(1)).send(any());

testSubject.request(2);
verify(stubDownstream, times(2)).send(any());
verify(stubDownstream, timeout(AWAIT_MILLIS).times(2)).send(any());

// there is 1 outstanding request for this message
upstream.offer(newMessage(), Context.empty());
verify(stubDownstream, times(3)).send(any());
verify(stubDownstream, timeout(AWAIT_MILLIS).times(3)).send(any());

testSubject.request(1);
// this message won't arrive
upstream.seal();

// still 3 messages sent
// still 3 messages sent, once the stream was seen to be completed
verify(stubDownstream, timeout(AWAIT_MILLIS)).complete();
verify(stubDownstream, times(3)).send(any());
verify(stubDownstream).complete();
}

@Test
Expand All @@ -144,8 +150,8 @@ void shouldSendLastMessageWithErrorWhenUpstreamCompletesExceptionally() {
// this should also trigger the completion
testSubject.request(1);

verify(stubDownstream, times(1)).send(any());
verify(stubDownstream).sendLast(assertArg(e -> {
verify(stubDownstream, timeout(AWAIT_MILLIS).times(1)).send(any());
verify(stubDownstream, timeout(AWAIT_MILLIS)).sendLast(assertArg(e -> {
assertThat(e.getErrorMessage().getMessage()).isEqualTo("Custom message");
assertThat(e.getErrorMessage().getErrorCode()).isEqualTo(ErrorCategory.QUERY_EXECUTION_ERROR.errorCode());
assertThat(e.getErrorCode()).isEqualTo(ErrorCategory.QUERY_EXECUTION_ERROR.errorCode());
Expand All @@ -164,24 +170,56 @@ void shouldSendLastMessageWithDetailsPayloadWhenUpstreamCompletesWithHandlerExec

detailsTestSubject.request(1);

verify(stubDownstream).sendLast(assertArg(e -> {
verify(stubDownstream, timeout(AWAIT_MILLIS)).sendLast(assertArg(e -> {
assertThat(e.getErrorMessage().getMessage()).isEqualTo("Custom message");
assertThat(e.getPayload().getData().toStringUtf8()).isEqualTo("Some details");
}));
}

@Test
void shouldEndTheQueryWithAnErrorAndCloseUpstreamWhenSendingAResponseFails() {
doThrow(new IllegalStateException("cannot send")).when(stubDownstream).send(any());
upstream.offer(newMessage(), Context.empty());

testSubject.request(1);

verify(stubDownstream, timeout(AWAIT_MILLIS)).sendLast(assertArg(e -> {
assertThat(e.getRequestIdentifier()).isEqualTo("test");
assertThat(e.getErrorMessage().getMessage()).isEqualTo("cannot send");
assertThat(e.getErrorCode()).isEqualTo(ErrorCategory.QUERY_EXECUTION_ERROR.errorCode());
}));
assertThat(upstream.isCompleted()).isTrue();
verify(stubDownstream, never()).complete();
}

@Test
void shouldReportAFailureWithoutDetailsWhenTheDetailsCannotBeConverted() {
doThrow(new IllegalStateException("cannot send")).when(stubDownstream).send(any());
doThrow(new IllegalArgumentException("cannot convert details")).doNothing().when(stubDownstream).sendLast(any());
upstream.offer(newMessage(), Context.empty());

testSubject.request(1);

ArgumentCaptor<QueryResponse> responses = ArgumentCaptor.forClass(QueryResponse.class);

verify(stubDownstream, timeout(AWAIT_MILLIS).times(2)).sendLast(responses.capture());

assertThat(responses.getAllValues().get(1).getErrorMessage().getMessage()).isEqualTo("cannot send");
assertThat(responses.getAllValues().get(1).hasPayload()).isFalse();
}

@Test
void shouldSendEmptySentinelResponseWhenUpstreamCompletesWithoutAnyMessage() {
upstream.seal();
testSubject.request(Long.MAX_VALUE);

verify(stubDownstream, never()).send(any());
verify(stubDownstream, never()).complete();
verify(stubDownstream).sendLast(assertArg(response -> {
verify(stubDownstream, timeout(AWAIT_MILLIS)).sendLast(assertArg(response -> {
assertThat(response.getRequestIdentifier()).isEqualTo("test");
assertThat(response.getPayload().getType()).isEqualTo("empty");
assertThat(response.getPayload().getData().isEmpty()).isTrue();
}));
verify(stubDownstream, never()).send(any());
verify(stubDownstream, never()).complete();
}

@Test
Expand Down Expand Up @@ -232,8 +270,10 @@ void shouldNeverLoseTheResponseWhenResultAndPermitsArriveConcurrently() throws E
}
}

// then
assertThat(sent.get()).isEqualTo(iterations);
assertThat(completed.get()).isEqualTo(iterations);
// then, once the sending that was handed off has caught up
await().atMost(Duration.ofSeconds(30)).untilAsserted(() -> {
assertThat(sent.get()).isEqualTo(iterations);
assertThat(completed.get()).isEqualTo(iterations);
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

package io.axoniq.framework.springcloud.query;

import io.axoniq.framework.messaging.distributed.CallbackHandOff;
import io.axoniq.framework.messaging.queryhandling.distributed.QueryBusConnector;
import io.axoniq.framework.messaging.queryhandling.distributed.QueryBusConnector.Handler;
import org.axonframework.common.FutureUtils;
Expand All @@ -29,6 +30,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Supplier;

/**
Expand Down Expand Up @@ -92,8 +94,8 @@ public void bind(QueryBusConnector.Handler handler) {
* Handles the given {@code request}, received from another member, writing its responses to the given
* {@code sink}.
* <p>
* Returns as soon as the query has been handed to the local handler. Responses are written to the {@code sink} as
* the handler produces them, which for a handler that answers asynchronously is after this method returned.
* Returns as soon as the query has been handed to the local handler. The responses are written to the
* {@code sink} later, on another thread, whenever the handler's stream has some.
*
* @param request the request received from another member
* @param sink receives the responses to the query, and the outcome that ends them
Expand Down Expand Up @@ -322,16 +324,20 @@ public CompletableFuture<Void> completeExceptionally(Throwable cause) {
/**
* Drains a response stream into a sink, as and when the stream has something to write.
* <p>
* The stream reports availability on whichever thread produced it, so draining is serialised here: two threads
* The stream calls back on the thread of whoever produced a response, which only schedules a drain on a thread of
* its own. Drains are serialised here: a signal arriving while one runs schedules the next, and two of them
* writing to one sink would interleave the responses of a single query.
*/
private class ResponseWriter {

private final String requestIdentifier;
private final MessageStream<QueryResponseMessage> responses;
private final QueryResponseSink sink;
// Volatile rather than guarded by this writer's monitor, so that cancelling does not wait on a drain that
// is blocked writing to a member which has already stopped reading.
// A drain blocks while writing to the sink, and a virtual thread that blocks inside a monitor holds on to its
// carrier thread on JDK 21, so drains are serialised with a lock.
private final ReentrantLock drainLock = new ReentrantLock();
// Volatile, so that cancelling does not wait on a drain that is blocked writing to a member which has already
// stopped reading.
private volatile boolean terminated = false;

private ResponseWriter(String requestIdentifier,
Expand All @@ -344,8 +350,9 @@ private ResponseWriter(String requestIdentifier,

private void start() {
sink.onUnavailable(this::cancel);
responses.setCallback(this::drain);
drain();
// The stream calls back on the thread of whoever produced a response, which may not read from it, or block.
// It also calls back at once when it has a response already, or has ended, so nothing is drained here.
responses.setCallback(CallbackHandOff.handOff(this::drain));
}

/**
Expand All @@ -363,34 +370,38 @@ private void cancel() {
responses.close();
}

private synchronized void drain() {
if (terminated) {
return;
}
private void drain() {
drainLock.lock();

try {
// Asked once per decision: a second ask can take a response that arrived in between, swallowing the
// notification that would have drained it, and leave it unwritten.
boolean hasNext = responses.hasNextAvailable();
while (hasNext) {
Optional<MessageStream.Entry<QueryResponseMessage>> next = responses.next();
if (next.isEmpty()) {
hasNext = false;
break;
}
sink.response(QueryConverter.convertResponseMessage(next.get().message(), requestIdentifier));
hasNext = responses.hasNextAvailable();
if (terminated) {
return;
}
// Reading a stream that has just ended reports its completion, which drains it again from inside
// this drain. That nested drain already finished the answer, so finishing it here would do so twice.
if (!hasNext && !terminated && responses.isCompleted()) {
responses.error().ifPresentOrElse(this::fail, this::finish);
try {
// Reads until the stream has nothing available: a response it reported and that is left unread
// is not announced by another callback.
while (responses.hasNextAvailable()) {
Optional<MessageStream.Entry<QueryResponseMessage>> next = responses.next();
if (next.isEmpty()) {
break;
}
sink.response(QueryConverter.convertResponseMessage(next.get().message(), requestIdentifier));
}
// Reading a stream that has just ended reports its completion, which schedules another drain.
// That one waits for this one and then finds the answer finished, so it is finished only once.
if (!terminated && responses.isCompleted()) {
responses.error().ifPresentOrElse(this::fail, this::finish);
}
} catch (Exception e) {
// Writing failed, which usually means the member that asked is gone. Reporting the failure back is
// pointless in that case, so the stream is simply released.
logger.debug("Could not write the responses to query [{}]; abandoning it.", requestIdentifier, e);
terminated = true;
responses.close();
}
} catch (Exception e) {
// Writing failed, which usually means the member that asked is gone. Reporting the failure back is
// pointless in that case, so the stream is simply released.
logger.debug("Could not write the responses to query [{}]; abandoning it.", requestIdentifier, e);
terminated = true;
responses.close();
}
finally {
drainLock.unlock();
}
}

Expand Down
Loading
Loading