perf: file variable ttl & quick create dataset with temp s3 bucket

newfish-cmyk · newfish-cmyk · commit 97ceee3e5c05 · 2025-11-25T10:08:21.000+08:00
diff --git a/packages/service/core/chat/saveChat.ts b/packages/service/core/chat/saveChat.ts
@@ -8,7 +8,7 @@ import { addLog } from '../../common/system/log';
 import { mongoSessionRun } from '../../common/mongo/sessionRun';
 import { type StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
 import { getAppChatConfig, getGuideModule } from '@fastgpt/global/core/workflow/utils';
-import { type AppChatConfigType } from '@fastgpt/global/core/app/type';
+import { type AppChatConfigType, type VariableItemType } from '@fastgpt/global/core/app/type';
 import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils';
 import { pushChatLog } from './pushChatLog';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
@@ -19,6 +19,7 @@ import { MongoChatItemResponse } from './chatItemResponseSchema';
 import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
 import type { ClientSession } from '../../common/mongo';
 import { removeS3TTL } from '../../common/s3/utils';
+import { VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
 
 type Props = {
   chatId: string;
@@ -51,27 +52,68 @@ const beforProcess = (props: Props) => {
 };
 const afterProcess = async ({
   contents,
+  variables,
+  variableList,
   session
 }: {
   contents: (UserChatItemType | AIChatItemType)[];
+  variables?: Record<string, any>;
+  variableList?: VariableItemType[];
   session: ClientSession;
 }) => {
-  const fileKeys = contents
+  const contentFileKeys = contents
     .map((item) => {
       if (item.value && Array.isArray(item.value)) {
-        return item.value.map((valueItem) => {
+        return item.value.flatMap((valueItem) => {
+          const keys: string[] = [];
+
+          // 1. chat file
           if (valueItem.type === ChatItemValueTypeEnum.file && valueItem.file?.key) {
-            return valueItem.file.key;
+            keys.push(valueItem.file.key);
+          }
+
+          // 2. plugin input
+          if (valueItem.type === 'text' && valueItem.text?.content) {
+            try {
+              const parsed = JSON.parse(valueItem.text.content);
+              if (Array.isArray(parsed)) {
+                parsed.forEach((field) => {
+                  if (field.value && Array.isArray(field.value)) {
+                    field.value.forEach((file: { key: string }) => {
+                      if (file.key && typeof file.key === 'string') {
+                        keys.push(file.key);
+                      }
+                    });
+                  }
+                });
+              }
+            } catch (err) {}
           }
+
+          return keys;
         });
       }
       return [];
     })
     .flat()
     .filter(Boolean) as string[];
 
-  if (fileKeys.length > 0) {
-    await removeS3TTL({ key: fileKeys, bucketName: 'private', session });
+  const variableFileKeys: string[] = [];
+  if (variables && variableList) {
+    variableList.forEach((varItem) => {
+      if (varItem.type === VariableInputEnum.file) {
+        const varValue = variables[varItem.key];
+        if (Array.isArray(varValue)) {
+          variableFileKeys.push(...varValue.map((item) => item.key));
+        }
+      }
+    });
+  }
+
+  const allFileKeys = [...new Set([...contentFileKeys, ...variableFileKeys])];
+
+  if (allFileKeys.length > 0) {
+    await removeS3TTL({ key: allFileKeys, bucketName: 'private', session });
   }
 };
 
@@ -254,7 +296,12 @@ export async function saveChat(props: Props) {
         }
       );
 
-      await afterProcess({ contents: processedContent, session });
+      await afterProcess({
+        contents: processedContent,
+        variables,
+        variableList,
+        session
+      });
 
       pushChatLog({
         chatId,
@@ -336,8 +383,18 @@ export async function saveChat(props: Props) {
 export const updateInteractiveChat = async (props: Props) => {
   beforProcess(props);
 
-  const { teamId, chatId, appId, userContent, aiContent, variables, durationSeconds, errorMsg } =
-    props;
+  const {
+    teamId,
+    chatId,
+    appId,
+    nodes,
+    appChatConfig,
+    userContent,
+    aiContent,
+    variables,
+    durationSeconds,
+    errorMsg
+  } = props;
   if (!chatId) return;
 
   const chatItem = await MongoChatItem.findOne({ appId, chatId, obj: ChatRoleEnum.AI }).sort({
@@ -371,6 +428,12 @@ export const updateInteractiveChat = async (props: Props) => {
     errorMsg
   });
 
+  const { variables: variableList } = getAppChatConfig({
+    chatConfig: appChatConfig,
+    systemConfigNode: getGuideModule(nodes),
+    isPublicFetch: false
+  });
+
   let finalInteractive = extractDeepestInteractive(interactiveValue.interactive);
 
   if (finalInteractive.type === 'userSelect') {
@@ -460,7 +523,12 @@ export const updateInteractiveChat = async (props: Props) => {
       );
     }
 
-    await afterProcess({ contents: [userContent, aiContent], session });
+    await afterProcess({
+      contents: [userContent, aiContent],
+      variables,
+      variableList,
+      session
+    });
   });
 
   // Push chat data logs
diff --git a/projects/app/src/pageComponents/app/detail/components/QuickCreateDatasetModal.tsx b/projects/app/src/pageComponents/app/detail/components/QuickCreateDatasetModal.tsx
@@ -19,14 +19,14 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { postCreateDatasetWithFiles, getDatasetById } from '@/web/core/dataset/api';
-import { getUploadAvatarPresignedUrl } from '@/web/common/file/api';
-import { uploadFile2DB } from '@/web/common/file/controller';
+import { getUploadAvatarPresignedUrl, getUploadTempFilePresignedUrl } from '@/web/common/file/api';
+import { POST } from '@/web/common/api/request';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { getWebDefaultEmbeddingModel, getWebDefaultLLMModel } from '@/web/common/system/utils';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { formatFileSize } from '@fastgpt/global/common/file/tools';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
+import { parseS3UploadError } from '@fastgpt/global/common/error/s3';
 import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io';
 import type { ImportSourceItemType } from '@/web/core/dataset/type';
 import FileSelector, {
@@ -82,11 +82,21 @@ const QuickCreateDatasetModal = ({
       await Promise.all(
         files.map(async ({ fileId, file }) => {
           try {
-            const { fileId: uploadFileId } = await uploadFile2DB({
-              file,
-              bucketName: BucketNameEnum.dataset,
-              data: { datasetId: '' },
-              percentListen: (percent) => {
+            const { url, fields, maxSize } = await getUploadTempFilePresignedUrl({
+              filename: file.name
+            });
+
+            const formData = new FormData();
+            Object.entries(fields).forEach(([k, v]) => formData.set(k, v));
+            formData.set('file', file);
+
+            await POST(url, formData, {
+              headers: {
+                'Content-Type': 'multipart/form-data; charset=utf-8'
+              },
+              onUploadProgress: (e) => {
+                if (!e.total) return;
+                const percent = Math.round((e.loaded / e.total) * 100);
                 setSelectFiles((state) =>
                   state.map((item) =>
                     item.id === fileId
@@ -100,20 +110,22 @@ const QuickCreateDatasetModal = ({
                   )
                 );
               }
-            });
-
-            setSelectFiles((state) =>
-              state.map((item) =>
-                item.id === fileId
-                  ? {
-                      ...item,
-                      dbFileId: uploadFileId,
-                      isUploading: false,
-                      uploadedFileRate: 100
-                    }
-                  : item
-              )
-            );
+            })
+              .then(() => {
+                setSelectFiles((state) =>
+                  state.map((item) =>
+                    item.id === fileId
+                      ? {
+                          ...item,
+                          dbFileId: fields.key,
+                          isUploading: false,
+                          uploadedFileRate: 100
+                        }
+                      : item
+                  )
+                );
+              })
+              .catch((error) => Promise.reject(parseS3UploadError({ t, error, maxSize })));
           } catch (error) {
             setSelectFiles((state) =>
               state.map((item) =>
diff --git a/projects/app/src/pages/api/common/file/presignTempFilePostUrl.ts b/projects/app/src/pages/api/common/file/presignTempFilePostUrl.ts
@@ -0,0 +1,44 @@
+import type { ApiRequestProps } from '@fastgpt/service/type/next';
+import { NextAPI } from '@/service/middleware/entry';
+import { type CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type';
+import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
+import { TeamDatasetCreatePermissionVal } from '@fastgpt/global/support/permission/user/constant';
+import { getFileS3Key } from '@fastgpt/service/common/s3/utils';
+import { S3PrivateBucket } from '@fastgpt/service/common/s3/buckets/private';
+import { authFrequencyLimit } from '@/service/common/frequencyLimit/api';
+import { addSeconds } from 'date-fns';
+
+export type PresignTempFilePostUrlParams = {
+  filename: string;
+};
+
+const authUploadLimit = (tmbId: string) => {
+  if (!global.feConfigs.uploadFileMaxAmount) return;
+  return authFrequencyLimit({
+    eventId: `${tmbId}-uploadfile`,
+    maxAmount: global.feConfigs.uploadFileMaxAmount * 2,
+    expiredTime: addSeconds(new Date(), 30) // 30s
+  });
+};
+
+async function handler(
+  req: ApiRequestProps<PresignTempFilePostUrlParams>
+): Promise<CreatePostPresignedUrlResult> {
+  const { filename } = req.body;
+
+  const { teamId, tmbId } = await authUserPer({
+    req,
+    authToken: true,
+    authApiKey: true,
+    per: TeamDatasetCreatePermissionVal
+  });
+
+  await authUploadLimit(tmbId);
+
+  const bucket = new S3PrivateBucket();
+  const { fileKey } = getFileS3Key.temp({ teamId, filename });
+
+  return await bucket.createPostPresignedUrl({ rawKey: fileKey, filename }, { expiredHours: 1 });
+}
+
+export default NextAPI(handler);
diff --git a/projects/app/src/pages/api/core/dataset/createWithFiles.ts b/projects/app/src/pages/api/core/dataset/createWithFiles.ts
@@ -33,10 +33,9 @@ import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
 import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
 import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar';
 import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
-import { getFileById, delFileByFileIdList } from '@fastgpt/service/common/file/gridfs/controller';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
-import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import type { EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
+import { S3PrivateBucket } from '@fastgpt/service/common/s3/buckets/private';
+import { getFileS3Key } from '@fastgpt/service/common/s3/utils';
 
 export type DatasetCreateWithFilesQuery = {};
 export type DatasetCreateWithFilesBody = {
@@ -124,16 +123,26 @@ async function handler(
       // 3. Refresh avatar
       await getS3AvatarSource().refreshAvatar(avatar, undefined, session);
 
-      // 4. Create collections for each file
+      // 4. Move temp files to dataset directory and create collections
+      const bucket = new S3PrivateBucket();
+
       for (const file of files) {
-        const gridFile = await getFileById({
-          bucketName: BucketNameEnum.dataset,
-          fileId: file.fileId
+        if (!file.fileId.startsWith('temp/')) {
+          return Promise.reject('Only temp files are supported');
+        }
+
+        const { fileKey: newKey } = getFileS3Key.dataset({
+          datasetId: String(dataset._id),
+          filename: file.name
         });
 
-        if (!gridFile) {
-          return Promise.reject(CommonErrEnum.fileNotFound);
-        }
+        await bucket.copy({
+          from: file.fileId,
+          to: newKey,
+          options: {
+            temporary: false
+          }
+        });
 
         await createCollectionAndInsertData({
           dataset,
@@ -142,10 +151,10 @@ async function handler(
             teamId,
             tmbId,
             type: DatasetCollectionTypeEnum.file,
-            name: file.name || gridFile.filename,
-            fileId: file.fileId,
+            name: file.name,
+            fileId: newKey,
             metadata: {
-              relatedImgId: file.fileId
+              relatedImgId: newKey
             },
             trainingType: DatasetCollectionDataProcessModeEnum.chunk,
             chunkTriggerType: ChunkTriggerConfigTypeEnum.minSize,
@@ -190,12 +199,6 @@ async function handler(
 
     return result;
   } catch (error) {
-    const fileIds = files.map((file) => file.fileId);
-    await delFileByFileIdList({
-      bucketName: BucketNameEnum.dataset,
-      fileIdList: fileIds
-    });
-
     return Promise.reject(error);
   }
 }
diff --git a/projects/app/src/web/common/file/api.ts b/projects/app/src/web/common/file/api.ts
@@ -61,3 +61,7 @@ export const getUploadDatasetFilePresignedUrl = (params: {
 }) => {
   return POST<CreatePostPresignedUrlResult>('/core/dataset/presignDatasetFilePostUrl', params);
 };
+
+export const getUploadTempFilePresignedUrl = (params: { filename: string }) => {
+  return POST<CreatePostPresignedUrlResult>('/common/file/presignTempFilePostUrl', params);
+};
