fix: text split code (#5773)

* fix: toolresponse result

* remove log

* stream remove

* fix: text split code

Author: c121914yu

document/content/docs/upgrading/4-13/4132.mdx

@@ -17,6 +17,7 @@ description: 'FastGPT V4.13.2 更新说明'
 
 1. LLM 模型默认支持图片，导致请求错误。
 2. Mongo 多副本切换时候，watch 未重新触发。
+3. 文本分块，所有策略用完后，未处理 LastText 数据。
 
 ## 🔨 插件更新
 

document/data/doc-last-modified.json

@@ -113,7 +113,7 @@
   "document/content/docs/upgrading/4-12/4124.mdx": "2025-09-17T22:29:56+08:00",
   "document/content/docs/upgrading/4-13/4130.mdx": "2025-09-30T16:00:10+08:00",
   "document/content/docs/upgrading/4-13/4131.mdx": "2025-09-30T15:47:06+08:00",
-  "document/content/docs/upgrading/4-13/4132.mdx": "2025-10-14T15:27:49+08:00",
+  "document/content/docs/upgrading/4-13/4132.mdx": "2025-10-17T13:58:27+08:00",
   "document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00",
   "document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00",
   "document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00",

packages/global/common/string/textSplitter.ts

@@ -295,17 +295,21 @@ const commonSplit = (props: SplitProps): SplitResponse => {
     const isMarkdownStep = checkIsMarkdownSplit(step);
     const isCustomStep = checkIsCustomStep(step);
     const forbidConcat = isCustomStep; // forbid=true时候，lastText肯定为空
-    const textLength = getTextValidLength(text);
 
     // Over step
     if (step >= stepReges.length) {
-      if (textLength < maxSize) {
-        return [text];
+      // Merge lastText with current text to prevent data loss
+      const combinedText = lastText + text;
+      const combinedLength = getTextValidLength(combinedText);
+
+      if (combinedLength < maxSize) {
+        return [combinedText];
       }
       // use slice-chunkSize to split text
+      // Note: Use combinedText.length for slicing, not combinedLength
       const chunks: string[] = [];
-      for (let i = 0; i < textLength; i += chunkSize - overlapLen) {
-        chunks.push(text.slice(i, i + chunkSize));
+      for (let i = 0; i < combinedText.length; i += chunkSize - overlapLen) {
+        chunks.push(combinedText.slice(i, i + chunkSize));
       }
       return chunks;
     }

projects/app/src/service/common/system/volumnMongoWatch.ts

@@ -7,7 +7,7 @@ import { getAppTemplatesAndLoadThem } from '@fastgpt/service/core/app/templates/
 import { watchSystemModelUpdate } from '@fastgpt/service/core/ai/config/utils';
 import { SystemConfigsTypeEnum } from '@fastgpt/global/common/system/config/constants';
 
-const changeStreams: any[] = [];
+let changeStreams: any[] = [];
 
 export const startMongoWatch = async () => {
   cleanupMongoWatch();
@@ -56,4 +56,5 @@ const cleanupMongoWatch = () => {
   changeStreams.forEach((changeStream) => {
     changeStream?.close();
   });
+  changeStreams = [];
 };

test/cases/global/common/string/textSplitter.test.ts

@@ -949,3 +949,37 @@ it(`Test splitText2Chunks 13 - Table split with empty lastText`, () => {
 
   expect(chunks).toEqual(mock.result);
 });
+
+// Test for lastText handling when all strategies exhausted (Issue #5770)
+it(`Test splitText2Chunks 14 - lastText not lost when strategies exhausted`, () => {
+  // This test verifies that when all splitting strategies are exhausted
+  // and forced character-based splitting occurs, lastText is not lost.
+  // The bug was: step >= stepReges.length returned [text] ignoring lastText
+
+  const mock = {
+    // Create text with NO good split points (no punctuation, newlines, etc.)
+    // This forces the algorithm to exhaust all strategies
+    text: 'A'.repeat(1800),
+    chunkSize: 500
+  };
+
+  const { chunks, chars } = splitText2Chunks({
+    text: mock.text,
+    chunkSize: mock.chunkSize,
+    overlapRatio: 0
+  });
+
+  // Critical test: No data loss - total characters in chunks should equal input
+  // This would fail with the bug because lastText would be dropped
+  // Even if the text is not split (treated as one chunk), data should not be lost
+  const totalCharsInChunks = chunks.join('').length;
+  expect(totalCharsInChunks).toBe(mock.text.length);
+
+  // Also verify the chars count is correct
+  expect(chars).toBe(mock.text.length);
+
+  // Verify no chunk is empty
+  chunks.forEach((chunk) => {
+    expect(chunk.length).toBeGreaterThan(0);
+  });
+});