[Pipe] Preserve downstream sink error messages (#18553)

Author: Caideyipi

iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java

@@ -2590,6 +2590,13 @@ private DataNodePipeMessages() {}
   public static final String MESSAGE_TRANSFER_FILE_ARG_ERROR_RESULT_STATUS_ARG_E565D9FD =
       "Transfer file %s error, result status %s.";
 
+  public static final String
+      EXCEPTION_FAILED_TO_RETRY_TRANSFERRING_EVENTS_IN_THE_RETRY_QUEUE_REMAINING_EVENTS_ARG_TABLET_EVENTS_ARG_TSFILE_EVENTS_ARG_5B4B2E7C =
+          "Failed to retry transferring events in the retry queue. Remaining events: %d (tablet events: %d, tsfile events: %d).";
+  public static final String
+      EXCEPTION_FAILED_TO_RETRY_TRANSFERRING_EVENTS_IN_THE_RETRY_QUEUE_REMAINING_EVENTS_ARG_TABLET_EVENTS_ARG_TSFILE_EVENTS_ARG_LAST_FAILURE_ARG_EB8F9DCD =
+          "Failed to retry transferring events in the retry queue. Remaining events: %d (tablet events: %d, tsfile events: %d). Last failure: %s.";
+
   public static final String EXCEPTION_LEGACY_PIPE_RECEIVER_REQUIRES_A_LOGGED_IN_SESSION_D96219BF =
       "Legacy pipe receiver requires a logged-in session.";
   public static final String EXCEPTION_FAILED_TO_SET_UP_CONSENSUS_SUBSCRIPTION_FOR_TOPIC_ARG_IN_CONSUMER_GROUP_ARG_ARG_A7FA88F3 =

iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java

@@ -2416,6 +2416,13 @@ private DataNodePipeMessages() {}
   public static final String MESSAGE_TRANSFER_FILE_ARG_ERROR_RESULT_STATUS_ARG_E565D9FD =
       "传输文件 %s 出错，结果状态为 %s。";
 
+  public static final String
+      EXCEPTION_FAILED_TO_RETRY_TRANSFERRING_EVENTS_IN_THE_RETRY_QUEUE_REMAINING_EVENTS_ARG_TABLET_EVENTS_ARG_TSFILE_EVENTS_ARG_5B4B2E7C =
+          "重试 retry queue 中的事件失败。剩余事件：%d（tablet 事件：%d，tsfile 事件：%d）。";
+  public static final String
+      EXCEPTION_FAILED_TO_RETRY_TRANSFERRING_EVENTS_IN_THE_RETRY_QUEUE_REMAINING_EVENTS_ARG_TABLET_EVENTS_ARG_TSFILE_EVENTS_ARG_LAST_FAILURE_ARG_EB8F9DCD =
+          "重试 retry queue 中的事件失败。剩余事件：%d（tablet 事件：%d，tsfile 事件：%d）。最近一次失败：%s。";
+
   public static final String EXCEPTION_LEGACY_PIPE_RECEIVER_REQUIRES_A_LOGGED_IN_SESSION_D96219BF =
       "Legacy pipe receiver 需要已登录的 session。";
   public static final String EXCEPTION_FAILED_TO_SET_UP_CONSENSUS_SUBSCRIPTION_FOR_TOPIC_ARG_IN_CONSUMER_GROUP_ARG_ARG_A7FA88F3 =

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java

@@ -34,6 +34,7 @@
 import org.apache.iotdb.commons.pipe.resource.log.PipeLogger;
 import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink;
 import org.apache.iotdb.commons.pipe.sink.protocol.PipeSinkWithSchedulingDelay;
+import org.apache.iotdb.commons.utils.ErrorHandlingCommonUtils;
 import org.apache.iotdb.db.i18n.DataNodePipeMessages;
 import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
 import org.apache.iotdb.db.pipe.event.common.deletion.PipeDeleteDataNodeEvent;
@@ -138,6 +139,8 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink implements PipeSinkWithS
   // Guarded by this. Events need identity semantics because the same payload may compare equal.
   private final Map<Event, PipeResourceFailureType> retryEvent2ResourceFailureType =
       new IdentityHashMap<>();
+  // Keep only the latest text to avoid retaining the complete exception chain for every event.
+  private volatile String lastRetryFailureMessage;
 
   private IoTDBDataNodeAsyncClientManager clientManager;
   private IoTDBDataNodeAsyncClientManager transferTsFileClientManager;
@@ -732,13 +735,11 @@ private void transferQueuedEventsIfNecessary(final boolean forced) {
 
         if (remainingEvents <= retryEventQueue.size() + retryTsFileQueue.size()) {
           final String message =
-              "Failed to retry transferring events in the retry queue. Remaining events: "
-                  + (retryEventQueue.size() + retryTsFileQueue.size())
-                  + " (tablet events: "
-                  + retryEventQueueEventCounter.getTabletInsertionEventCount()
-                  + ", tsfile events: "
-                  + retryEventQueueEventCounter.getTsFileInsertionEventCount()
-                  + ").";
+              formatRetryQueueFailureMessage(
+                  retryEventQueue.size() + retryTsFileQueue.size(),
+                  retryEventQueueEventCounter.getTabletInsertionEventCount(),
+                  retryEventQueueEventCounter.getTsFileInsertionEventCount(),
+                  lastRetryFailureMessage);
           final PipeResourceFailureType retryQueueResourceFailureType =
               getRetryQueueResourceFailureType();
           if (retryQueueResourceFailureType != null) {
@@ -751,6 +752,12 @@ private void transferQueuedEventsIfNecessary(final boolean forced) {
         }
       }
     }
+
+    synchronized (this) {
+      if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
+        lastRetryFailureMessage = null;
+      }
+    }
   }
 
   private void retryTransfer(final TabletInsertionEvent tabletInsertionEvent) {
@@ -834,6 +841,14 @@ private synchronized void addFailureEventToRetryQueue(
       return;
     }
 
+    if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
+      lastRetryFailureMessage = null;
+    }
+
+    if (e != null) {
+      lastRetryFailureMessage = getRetryFailureMessage(e);
+    }
+
     if (resourceFailureType != null && event instanceof EnrichedEvent) {
       final EnrichedEvent enrichedEvent = (EnrichedEvent) event;
       final Pair<String, Long> pipeKey =
@@ -881,6 +896,46 @@ public void addFailureEventsToRetryQueue(
     events.forEach(event -> addFailureEventToRetryQueue(event, e, failureRecordedPipes));
   }
 
+  static String formatRetryQueueFailureMessage(
+      final int remainingEvents,
+      final int tabletEventCount,
+      final int tsFileEventCount,
+      final String lastFailureMessage) {
+    if (!hasText(lastFailureMessage)) {
+      return String.format(
+          DataNodePipeMessages
+              .EXCEPTION_FAILED_TO_RETRY_TRANSFERRING_EVENTS_IN_THE_RETRY_QUEUE_REMAINING_EVENTS_ARG_TABLET_EVENTS_ARG_TSFILE_EVENTS_ARG_5B4B2E7C,
+          remainingEvents,
+          tabletEventCount,
+          tsFileEventCount);
+    }
+    return String.format(
+        DataNodePipeMessages
+            .EXCEPTION_FAILED_TO_RETRY_TRANSFERRING_EVENTS_IN_THE_RETRY_QUEUE_REMAINING_EVENTS_ARG_TABLET_EVENTS_ARG_TSFILE_EVENTS_ARG_LAST_FAILURE_ARG_EB8F9DCD,
+        remainingEvents,
+        tabletEventCount,
+        tsFileEventCount,
+        lastFailureMessage);
+  }
+
+  private static String getRetryFailureMessage(final Exception exception) {
+    final Throwable rootCause = ErrorHandlingCommonUtils.getRootCause(exception);
+    if (hasText(rootCause.getMessage())) {
+      return rootCause.getMessage();
+    }
+    // Throwable#getMessage() is null for exceptions such as a bare NPE. Keep the type in the
+    // reported sink error instead of falling back to a generic transfer wrapper.
+    return rootCause.toString();
+  }
+
+  private static boolean hasText(final String message) {
+    return message != null && !message.trim().isEmpty();
+  }
+
+  synchronized String getLastRetryFailureMessage() {
+    return lastRetryFailureMessage;
+  }
+
   private synchronized PipeResourceFailureType getRetryQueueResourceFailureType() {
     for (final PipeResourceFailureType failureType : PipeResourceFailureType.values()) {
       if (retryEvent2ResourceFailureType.containsValue(failureType)) {
@@ -1056,6 +1111,10 @@ && isDroppedPipe((EnrichedEvent) event, committerKey)) {
           }
           return false;
         });
+
+    if (retryEventQueue.isEmpty() && retryTsFileQueue.isEmpty()) {
+      lastRetryFailureMessage = null;
+    }
   }
 
   @Override
@@ -1110,6 +1169,7 @@ public synchronized void clearRetryEventsReferenceCount() {
       }
     }
     retryEvent2ResourceFailureType.clear();
+    lastRetryFailureMessage = null;
   }
 
   //////////////////////// APIs provided for metric framework ////////////////////////

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/PipeStatementTsStatusVisitorTest.java

@@ -66,6 +66,24 @@ StatusUtils.OK, new TSStatus(TSStatusCode.OUT_OF_TTL.getStatusCode()))))
             .getCode());
   }
 
+  @Test
+  public void testMultipleErrorPropagatesSelectedReceiverMessage() {
+    final TSStatus status =
+        IoTDBDataNodeReceiver.STATEMENT_STATUS_VISITOR.process(
+            new InsertRowsStatement(),
+            new TSStatus(TSStatusCode.MULTIPLE_ERROR.getStatusCode())
+                .setSubStatus(
+                    Arrays.asList(
+                        StatusUtils.OK,
+                        new TSStatus(TSStatusCode.WRITE_PROCESS_REJECT.getStatusCode())
+                            .setMessage("receiver write queue is full"))));
+
+    Assert.assertEquals(
+        TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode(),
+        status.getCode());
+    Assert.assertEquals("receiver write queue is full", status.getMessage());
+  }
+
   @Test
   public void testLoadTemporaryUnavailableClassification() throws Exception {
     final File tsFile = File.createTempFile("temporary-unavailable", ".tsfile");

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSinkTest.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.sink.protocol.thrift.async;
+
+import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class IoTDBDataRegionAsyncSinkTest {
+
+  @Test
+  public void testRetryQueueFailureMessageIncludesRootCauseAndIsCleared() {
+    final IoTDBDataRegionAsyncSink sink = new IoTDBDataRegionAsyncSink();
+    final Event event = Mockito.mock(Event.class);
+
+    sink.addFailureEventToRetryQueue(
+        event,
+        new PipeException(
+            "sink transfer wrapper", new IllegalStateException("receiver rejected request")));
+
+    Assert.assertEquals("receiver rejected request", sink.getLastRetryFailureMessage());
+    Assert.assertTrue(
+        IoTDBDataRegionAsyncSink.formatRetryQueueFailureMessage(
+                1, 1, 0, sink.getLastRetryFailureMessage())
+            .contains("receiver rejected request"));
+
+    sink.clearRetryEventsReferenceCount();
+
+    Assert.assertNull(sink.getLastRetryFailureMessage());
+    Assert.assertFalse(
+        IoTDBDataRegionAsyncSink.formatRetryQueueFailureMessage(
+                0, 0, 0, sink.getLastRetryFailureMessage())
+            .contains("receiver rejected request"));
+  }
+
+  @Test
+  public void testRetryQueueFailureMessageKeepsRootCauseTypeWhenMessageIsMissing() {
+    final IoTDBDataRegionAsyncSink sink = new IoTDBDataRegionAsyncSink();
+    final Event event = Mockito.mock(Event.class);
+
+    sink.addFailureEventToRetryQueue(
+        event, new PipeException("sink transfer wrapper", new NullPointerException()));
+
+    Assert.assertEquals("java.lang.NullPointerException", sink.getLastRetryFailureMessage());
+  }
+}

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandlerCleanupTest.java

@@ -19,15 +19,21 @@
 
 package org.apache.iotdb.db.pipe.sink.protocol.thrift.async.handler;
 
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
 import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
+import org.apache.iotdb.commons.pipe.receiver.PipeReceiverStatusHandler;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp;
 
 import org.junit.Assert;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.io.File;
+import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.util.Collections;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -70,6 +76,56 @@ public void testCloseKeepsSourceTsFile() throws Exception {
     }
   }
 
+  @Test
+  public void testSealFailurePassesNestedReceiverMessageToRetryQueue() throws Exception {
+    final File file = Files.createTempFile("pipe-transfer-seal-failure", ".tsfile").toFile();
+    try {
+      final PipeTsFileInsertionEvent event = Mockito.mock(PipeTsFileInsertionEvent.class);
+      final IoTDBDataRegionAsyncSink sink = Mockito.mock(IoTDBDataRegionAsyncSink.class);
+      Mockito.when(sink.statusHandler())
+          .thenReturn(new PipeReceiverStatusHandler(false, 60, false, 60, false, false));
+
+      final PipeTransferTsFileHandler handler =
+          new PipeTransferTsFileHandler(
+              sink,
+              Collections.emptyMap(),
+              Collections.singletonList(event),
+              new AtomicInteger(1),
+              new AtomicBoolean(false),
+              file,
+              null,
+              false,
+              null);
+      markSealSignalSent(handler);
+
+      final TSStatus status =
+          new TSStatus(TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode())
+              .setMessage("aggregate load failure")
+              .setSubStatus(
+                  Collections.singletonList(
+                      new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode())
+                          .setMessage("receiver disk is full")));
+
+      Assert.assertFalse(handler.onCompleteInternal(new TPipeTransferResp(status)));
+
+      final ArgumentCaptor<Exception> exceptionCaptor = ArgumentCaptor.forClass(Exception.class);
+      Mockito.verify(sink)
+          .addFailureEventsToRetryQueue(
+              Mockito.eq(Collections.singletonList(event)), exceptionCaptor.capture());
+      Assert.assertEquals("receiver disk is full", exceptionCaptor.getValue().getMessage());
+    } finally {
+      if (file.exists()) {
+        Assert.assertTrue(file.delete());
+      }
+    }
+  }
+
+  private static void markSealSignalSent(final PipeTransferTsFileHandler handler) throws Exception {
+    final Field field = PipeTransferTsFileHandler.class.getDeclaredField("isSealSignalSent");
+    field.setAccessible(true);
+    ((AtomicBoolean) field.get(handler)).set(true);
+  }
+
   private PipeTransferTsFileHandler createHandler(final File file, final EnrichedEvent event)
       throws Exception {
     return new PipeTransferTsFileHandler(