perf: file variable ttl & quick create dataset with temp s3 bucket

packages/global/core/workflow/constants.ts

@@ -391,12 +391,6 @@ export const variableConfigs: VariableConfigType[][] = [
       label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
       value: VariableInputEnum.timeRangeSelect,
       defaultValueType: WorkflowIOValueTypeEnum.arrayString
-    },
-    {
-      icon: 'core/workflow/inputType/file',
-      label: i18nT('common:core.workflow.inputType.file'),
-      value: VariableInputEnum.file,
-      defaultValueType: WorkflowIOValueTypeEnum.arrayString
     }
   ],
   [
@@ -407,10 +401,10 @@ export const variableConfigs: VariableConfigType[][] = [
       defaultValueType: WorkflowIOValueTypeEnum.string
     },
     {
-      icon: 'core/workflow/inputType/dataset',
-      label: i18nT('common:core.workflow.inputType.datasetSelect'),
-      value: VariableInputEnum.datasetSelect,
-      defaultValueType: WorkflowIOValueTypeEnum.selectDataset
+      icon: 'core/workflow/inputType/file',
+      label: i18nT('common:core.workflow.inputType.file'),
+      value: VariableInputEnum.file,
+      defaultValueType: WorkflowIOValueTypeEnum.arrayString
     }
   ],
   [

packages/service/common/file/gridfs/controller.ts

@@ -82,59 +82,6 @@ export async function uploadFile({
 
   return String(stream.id);
 }
-export async function uploadFileFromBase64Img({
-  bucketName,
-  teamId,
-  tmbId,
-  base64,
-  filename,
-  metadata = {}
-}: {
-  bucketName: `${BucketNameEnum}`;
-  teamId: string;
-  tmbId: string;
-  base64: string;
-  filename: string;
-  metadata?: Record<string, any>;
-}) {
-  if (!base64) return Promise.reject(`filePath is empty`);
-  if (!filename) return Promise.reject(`filename is empty`);
-
-  const base64Data = base64.split(',')[1];
-  const contentType = base64.split(',')?.[0]?.split?.(':')?.[1];
-  const buffer = Buffer.from(base64Data, 'base64');
-  const readableStream = new Readable({
-    read() {
-      this.push(buffer);
-      this.push(null);
-    }
-  });
-
-  const { stream: readStream, encoding } = await stream2Encoding(readableStream);
-
-  // Add default metadata
-  metadata.teamId = teamId;
-  metadata.tmbId = tmbId;
-  metadata.encoding = encoding;
-
-  // create a gridfs bucket
-  const bucket = getGridBucket(bucketName);
-
-  const stream = bucket.openUploadStream(filename, {
-    metadata,
-    contentType
-  });
-
-  // save to gridfs
-  await new Promise((resolve, reject) => {
-    readStream
-      .pipe(stream as any)
-      .on('finish', resolve)
-      .on('error', reject);
-  });
-
-  return String(stream.id);
-}
 
 export async function getFileById({
   bucketName,

packages/service/common/s3/buckets/base.ts

@@ -75,7 +75,15 @@ export class S3BaseBucket {
   }
 
   // TODO: 加到 MQ 里保障幂等
-  async move(from: string, to: string, options: CopyConditions): Promise<void> {
+  async move({
+    from,
+    to,
+    options
+  }: {
+    from: string;
+    to: string;
+    options?: CopyConditions;
+  }): Promise<void> {
     await this.copy({
       from,
       to,

packages/service/core/chat/saveChat.ts

@@ -8,7 +8,7 @@ import { addLog } from '../../common/system/log';
 import { mongoSessionRun } from '../../common/mongo/sessionRun';
 import { type StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
 import { getAppChatConfig, getGuideModule } from '@fastgpt/global/core/workflow/utils';
-import { type AppChatConfigType } from '@fastgpt/global/core/app/type';
+import { type AppChatConfigType, type VariableItemType } from '@fastgpt/global/core/app/type';
 import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils';
 import { pushChatLog } from './pushChatLog';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
@@ -19,6 +19,7 @@ import { MongoChatItemResponse } from './chatItemResponseSchema';
 import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
 import type { ClientSession } from '../../common/mongo';
 import { removeS3TTL } from '../../common/s3/utils';
+import { VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
 
 type Props = {
   chatId: string;
@@ -51,27 +52,68 @@ const beforProcess = (props: Props) => {
 };
 const afterProcess = async ({
   contents,
+  variables,
+  variableList,
   session
 }: {
   contents: (UserChatItemType | AIChatItemType)[];
+  variables?: Record<string, any>;
+  variableList?: VariableItemType[];
   session: ClientSession;
 }) => {
-  const fileKeys = contents
+  const contentFileKeys = contents
     .map((item) => {
       if (item.value && Array.isArray(item.value)) {
-        return item.value.map((valueItem) => {
+        return item.value.flatMap((valueItem) => {
+          const keys: string[] = [];
+
+          // 1. chat file
           if (valueItem.type === ChatItemValueTypeEnum.file && valueItem.file?.key) {
-            return valueItem.file.key;
+            keys.push(valueItem.file.key);
+          }
+
+          // 2. plugin input
+          if (valueItem.type === 'text' && valueItem.text?.content) {
+            try {
+              const parsed = JSON.parse(valueItem.text.content);
+              if (Array.isArray(parsed)) {
+                parsed.forEach((field) => {
+                  if (field.value && Array.isArray(field.value)) {
+                    field.value.forEach((file: { key: string }) => {
+                      if (file.key && typeof file.key === 'string') {
+                        keys.push(file.key);
+                      }
+                    });
+                  }
+                });
+              }
+            } catch (err) {}
           }
+
+          return keys;
         });
       }
       return [];
     })
     .flat()
     .filter(Boolean) as string[];
 
-  if (fileKeys.length > 0) {
-    await removeS3TTL({ key: fileKeys, bucketName: 'private', session });
+  const variableFileKeys: string[] = [];
+  if (variables && variableList) {
+    variableList.forEach((varItem) => {
+      if (varItem.type === VariableInputEnum.file) {
+        const varValue = variables[varItem.key];
+        if (Array.isArray(varValue)) {
+          variableFileKeys.push(...varValue.map((item) => item.key));
+        }
+      }
+    });
+  }
+
+  const allFileKeys = [...new Set([...contentFileKeys, ...variableFileKeys])];
+
+  if (allFileKeys.length > 0) {
+    await removeS3TTL({ key: allFileKeys, bucketName: 'private', session });
   }
 };
 
@@ -254,7 +296,12 @@ export async function saveChat(props: Props) {
         }
       );
 
-      await afterProcess({ contents: processedContent, session });
+      await afterProcess({
+        contents: processedContent,
+        variables,
+        variableList,
+        session
+      });
 
       pushChatLog({
         chatId,
@@ -336,8 +383,18 @@ export async function saveChat(props: Props) {
 export const updateInteractiveChat = async (props: Props) => {
   beforProcess(props);
 
-  const { teamId, chatId, appId, userContent, aiContent, variables, durationSeconds, errorMsg } =
-    props;
+  const {
+    teamId,
+    chatId,
+    appId,
+    nodes,
+    appChatConfig,
+    userContent,
+    aiContent,
+    variables,
+    durationSeconds,
+    errorMsg
+  } = props;
   if (!chatId) return;
 
   const chatItem = await MongoChatItem.findOne({ appId, chatId, obj: ChatRoleEnum.AI }).sort({
@@ -371,6 +428,12 @@ export const updateInteractiveChat = async (props: Props) => {
     errorMsg
   });
 
+  const { variables: variableList } = getAppChatConfig({
+    chatConfig: appChatConfig,
+    systemConfigNode: getGuideModule(nodes),
+    isPublicFetch: false
+  });
+
   let finalInteractive = extractDeepestInteractive(interactiveValue.interactive);
 
   if (finalInteractive.type === 'userSelect') {
@@ -460,7 +523,12 @@ export const updateInteractiveChat = async (props: Props) => {
       );
     }
 
-    await afterProcess({ contents: [userContent, aiContent], session });
+    await afterProcess({
+      contents: [userContent, aiContent],
+      variables,
+      variableList,
+      session
+    });
   });
 
   // Push chat data logs

projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodeVariableUpdate.tsx

@@ -6,7 +6,11 @@ import { useTranslation } from 'next-i18next';
 import { Box, Button, Flex } from '@chakra-ui/react';
 import { type TUpdateListItem } from '@fastgpt/global/core/workflow/template/system/variableUpdate/type';
 import type { WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants';
-import { NodeInputKeyEnum, VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants';
+import {
+  NodeInputKeyEnum,
+  VARIABLE_NODE_ID,
+  VariableInputEnum
+} from '@fastgpt/global/core/workflow/constants';
 import { useContextSelector } from 'use-context-selector';
 import {
   FlowNodeInputMap,
@@ -120,8 +124,14 @@ const NodeVariableUpdate = ({ data, selected }: NodeProps<FlowNodeItemType>) =>
           const variableList = appDetail.chatConfig.variables || [];
           const variable = variableList.find((item) => item.key === value[1]);
           if (variable) {
+            // 文件类型在变量更新节点中使用文本框,因为不在运行时上下文中,无法使用文件选择器
+            const inputType =
+              variable.type === VariableInputEnum.file
+                ? InputTypeEnum.textarea
+                : variableInputTypeToInputType(variable.type);
+
             return {
-              inputType: variableInputTypeToInputType(variable.type),
+              inputType,
               formParams: {
                 // 获取变量中一些表单配置
                 maxLength: variable.maxLength,

projects/app/src/pageComponents/app/detail/components/QuickCreateDatasetModal.tsx

@@ -19,14 +19,14 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useUploadAvatar } from '@fastgpt/web/common/file/hooks/useUploadAvatar';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { postCreateDatasetWithFiles, getDatasetById } from '@/web/core/dataset/api';
-import { getUploadAvatarPresignedUrl } from '@/web/common/file/api';
-import { uploadFile2DB } from '@/web/common/file/controller';
+import { getUploadAvatarPresignedUrl, getUploadTempFilePresignedUrl } from '@/web/common/file/api';
+import { POST } from '@/web/common/api/request';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { getWebDefaultEmbeddingModel, getWebDefaultLLMModel } from '@/web/common/system/utils';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { formatFileSize } from '@fastgpt/global/common/file/tools';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
+import { parseS3UploadError } from '@fastgpt/global/common/error/s3';
 import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io';
 import type { ImportSourceItemType } from '@/web/core/dataset/type';
 import FileSelector, {
@@ -82,11 +82,21 @@ const QuickCreateDatasetModal = ({
       await Promise.all(
         files.map(async ({ fileId, file }) => {
           try {
-            const { fileId: uploadFileId } = await uploadFile2DB({
-              file,
-              bucketName: BucketNameEnum.dataset,
-              data: { datasetId: '' },
-              percentListen: (percent) => {
+            const { url, fields, maxSize } = await getUploadTempFilePresignedUrl({
+              filename: file.name
+            });
+
+            const formData = new FormData();
+            Object.entries(fields).forEach(([k, v]) => formData.set(k, v));
+            formData.set('file', file);
+
+            await POST(url, formData, {
+              headers: {
+                'Content-Type': 'multipart/form-data; charset=utf-8'
+              },
+              onUploadProgress: (e) => {
+                if (!e.total) return;
+                const percent = Math.round((e.loaded / e.total) * 100);
                 setSelectFiles((state) =>
                   state.map((item) =>
                     item.id === fileId
@@ -100,20 +110,22 @@ const QuickCreateDatasetModal = ({
                   )
                 );
               }
-            });
-
-            setSelectFiles((state) =>
-              state.map((item) =>
-                item.id === fileId
-                  ? {
-                      ...item,
-                      dbFileId: uploadFileId,
-                      isUploading: false,
-                      uploadedFileRate: 100
-                    }
-                  : item
-              )
-            );
+            })
+              .then(() => {
+                setSelectFiles((state) =>
+                  state.map((item) =>
+                    item.id === fileId
+                      ? {
+                          ...item,
+                          dbFileId: fields.key,
+                          isUploading: false,
+                          uploadedFileRate: 100
+                        }
+                      : item
+                  )
+                );
+              })
+              .catch((error) => Promise.reject(parseS3UploadError({ t, error, maxSize })));
           } catch (error) {
             setSelectFiles((state) =>
               state.map((item) =>

projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx

@@ -9,8 +9,6 @@ import { RenderUploadFiles } from '../components/RenderFiles';
 import { useContextSelector } from 'use-context-selector';
 import { DatasetImportContext } from '../Context';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { uploadFile2DB } from '@/web/common/file/controller';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { formatFileSize } from '@fastgpt/global/common/file/tools';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';