diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 4f54bf170d..0bf3c23ab0 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -73,6 +73,7 @@ "@xstate/react": "^6.0.0", "@xstate/store": "^3.11.2", "ai": "^5.0.93", + "async-mutex": "^0.5.0", "chroma-js": "^3.1.2", "clsx": "^2.1.1", "date-fns": "^4.1.0", diff --git a/apps/desktop/src-tauri/capabilities/default.json b/apps/desktop/src-tauri/capabilities/default.json index 8d69e94915..1cd05a0bd8 100644 --- a/apps/desktop/src-tauri/capabilities/default.json +++ b/apps/desktop/src-tauri/capabilities/default.json @@ -69,6 +69,12 @@ { "path": "$DOWNLOAD/**/*" } ] }, + { + "identifier": "fs:allow-remove", + "allow": [ + { "path": "$DATA/hyprnote/**/*" } + ] + }, "db2:default", "windows:default", "tracing:default", diff --git a/apps/desktop/src/chat/resolve-attachments.ts b/apps/desktop/src/chat/resolve-attachments.ts new file mode 100644 index 0000000000..d3d8ca8a7a --- /dev/null +++ b/apps/desktop/src/chat/resolve-attachments.ts @@ -0,0 +1,52 @@ +import type { FileUIPart } from "ai"; + +import { readChatAttachmentAsDataURL } from "../components/chat/attachments/storage"; +import type { HyprUIMessage } from "./types"; + +export async function resolveChatFileReferences( + messages: HyprUIMessage[], + chatGroupId?: string, +): Promise { + const resolved: HyprUIMessage[] = []; + + for (const message of messages) { + const resolvedParts = await Promise.all( + message.parts.map(async (part) => { + if (part.type === "data-chat-file") { + if (!chatGroupId) { + return part; + } + + const dataUrl = await readChatAttachmentAsDataURL( + chatGroupId, + part.data.attachmentId, + ); + + if (!dataUrl) { + return part; + } + + return { + type: "file", + filename: part.data.filename, + mediaType: part.data.mediaType, + url: dataUrl, + } satisfies FileUIPart; + } + + if (part.type === "file" && part.url.startsWith("blob:")) { + return part; + } + + return part; + }), + ); + + resolved.push({ + ...message, + parts: resolvedParts, + }); + } + + return resolved; +} diff --git a/apps/desktop/src/chat/transport.ts b/apps/desktop/src/chat/transport.ts index 5f89183db1..600a4ffdc1 100644 --- a/apps/desktop/src/chat/transport.ts +++ b/apps/desktop/src/chat/transport.ts @@ -7,12 +7,14 @@ import { } from "ai"; import { type ToolRegistry } from "../contexts/tool"; +import { resolveChatFileReferences } from "./resolve-attachments"; import type { HyprUIMessage } from "./types"; export class CustomChatTransport implements ChatTransport { constructor( private registry: ToolRegistry, private model: LanguageModel, + private chatGroupId?: string, ) {} sendMessages: ChatTransport["sendMessages"] = async ( @@ -20,6 +22,11 @@ export class CustomChatTransport implements ChatTransport { ) => { const tools = this.registry.getTools("chat"); + const resolvedMessages = await resolveChatFileReferences( + options.messages, + this.chatGroupId, + ); + const agent = new Agent({ model: this.model, tools, @@ -34,7 +41,7 @@ export class CustomChatTransport implements ChatTransport { }); const result = agent.stream({ - messages: convertToModelMessages(options.messages), + messages: convertToModelMessages(resolvedMessages), }); return result.toUIMessageStream({ diff --git a/apps/desktop/src/chat/types.ts b/apps/desktop/src/chat/types.ts index f8db872586..7f2498346a 100644 --- a/apps/desktop/src/chat/types.ts +++ b/apps/desktop/src/chat/types.ts @@ -6,4 +6,18 @@ export const messageMetadataSchema = z.object({ }); export type MessageMetadata = z.infer; -export type HyprUIMessage = UIMessage; + +export type ChatFileReferencePart = { + type: "chat-file"; + attachmentId: string; + filename: string; + mediaType: string; + size: number; + fileUrl: string; +}; + +export type ChatDataParts = { + "chat-file": ChatFileReferencePart; +}; + +export type HyprUIMessage = UIMessage; diff --git a/apps/desktop/src/components/chat/attachments/storage.ts b/apps/desktop/src/components/chat/attachments/storage.ts new file mode 100644 index 0000000000..08362e916e --- /dev/null +++ b/apps/desktop/src/components/chat/attachments/storage.ts @@ -0,0 +1,52 @@ +import { ATTACHMENT_SIZE_LIMIT } from "../../../shared/attachments/constants"; +import { + createAttachmentStorage, + ManifestCorruptionError, + type ManifestEntry, +} from "../../../shared/attachments/storage"; + +export type PersistedChatAttachment = ManifestEntry & { + filePath: string; + fileUrl: string; +}; + +export { ManifestCorruptionError }; + +const chatStorage = createAttachmentStorage({ + getBasePath: (groupId: string) => `hyprnote/chat/${groupId}`, + entityName: "chat group", + maxSize: ATTACHMENT_SIZE_LIMIT, + includeTitle: false, +}); + +export async function loadChatAttachments( + groupId: string, +): Promise { + return await chatStorage.load(groupId); +} + +export async function saveChatAttachment( + groupId: string, + file: File, + attachmentId = crypto.randomUUID(), +): Promise { + return await chatStorage.save(groupId, file, {}, attachmentId); +} + +export async function removeChatAttachment( + groupId: string, + attachmentId: string, +) { + return await chatStorage.remove(groupId, attachmentId); +} + +export async function removeChatGroupAttachments(groupId: string) { + return await chatStorage.removeAll(groupId); +} + +export async function readChatAttachmentAsDataURL( + groupId: string, + attachmentId: string, +): Promise { + return await chatStorage.readAsDataURL(groupId, attachmentId); +} diff --git a/apps/desktop/src/components/chat/input.tsx b/apps/desktop/src/components/chat/input.tsx index 1c55265949..4405f5615e 100644 --- a/apps/desktop/src/components/chat/input.tsx +++ b/apps/desktop/src/components/chat/input.tsx @@ -1,5 +1,18 @@ -import { FullscreenIcon, MicIcon, PaperclipIcon, SendIcon } from "lucide-react"; -import { useCallback, useEffect, useRef } from "react"; +import type { FileUIPart, TextUIPart } from "ai"; +import { + FullscreenIcon, + MicIcon, + PaperclipIcon, + SendIcon, + X, +} from "lucide-react"; +import { + type ChangeEvent, + useCallback, + useEffect, + useRef, + useState, +} from "react"; import type { TiptapEditor } from "@hypr/tiptap/editor"; import Editor from "@hypr/tiptap/editor"; @@ -10,31 +23,129 @@ import { import { Button } from "@hypr/ui/components/ui/button"; import { cn } from "@hypr/utils"; +import type { HyprUIMessage } from "../../chat/types"; import { useShell } from "../../contexts/shell"; +import { useCurrentModelModalitySupport } from "../../hooks/useCurrentModelModalitySupport"; +import { ATTACHMENT_SIZE_LIMIT } from "../../shared/attachments/constants"; +import { + type PersistedChatAttachment, + removeChatAttachment, + saveChatAttachment, +} from "./attachments/storage"; + +type MessagePart = TextUIPart | FileUIPart | HyprUIMessage["parts"][number]; export function ChatMessageInput({ onSendMessage, disabled: disabledProp, + chatGroupId, }: { - onSendMessage: (content: string, parts: any[]) => void; + onSendMessage: ( + content: string, + parts: MessagePart[], + attachments: Array<{ + file: File; + persisted?: PersistedChatAttachment; + }>, + ) => Promise | void; disabled?: boolean | { disabled: boolean; message?: string }; + chatGroupId?: string; }) { const editorRef = useRef<{ editor: TiptapEditor | null }>(null); + const fileInputRef = useRef(null); + const [attachedFiles, setAttachedFiles] = useState< + Array<{ file: File; persisted?: PersistedChatAttachment }> + >([]); + const [attachmentError, setAttachmentError] = useState(null); + const modalitySupport = useCurrentModelModalitySupport(); + + const hasImageAttachments = attachedFiles.some((item) => + isImageFile(item.file), + ); + const capabilityWarning = + hasImageAttachments && modalitySupport && !modalitySupport.includes("image") + ? "This model does not support images. These attachments will be ignored." + : null; const disabled = typeof disabledProp === "object" ? disabledProp.disabled : disabledProp; - const handleSubmit = useCallback(() => { + const handleSubmit = useCallback(async () => { const json = editorRef.current?.editor?.getJSON(); const text = tiptapJsonToText(json).trim(); - if (!text || disabled) { + if (disabled) { + return; + } + + const filteredFiles = attachedFiles.filter((item) => { + if (!modalitySupport) { + return true; + } + + const isImage = isImageFile(item.file); + + if (isImage && !modalitySupport.includes("image")) { + return false; + } + + return true; + }); + + if (!text && filteredFiles.length === 0) { + return; + } + + const fileParts: MessagePart[] = filteredFiles.map((item) => { + if (item.persisted) { + return { + type: "data-chat-file", + data: { + type: "chat-file", + attachmentId: item.persisted.id, + filename: item.persisted.fileName, + mediaType: item.persisted.mimeType, + size: item.persisted.size, + fileUrl: item.persisted.fileUrl, + }, + }; + } + + return { + type: "file", + filename: item.file.name, + mediaType: item.file.type, + url: URL.createObjectURL(item.file), + } satisfies FileUIPart; + }); + + const parts: MessagePart[] = [ + ...(text ? [{ type: "text" as const, text }] : []), + ...fileParts, + ]; + + const attachmentsForMessage = filteredFiles.map(({ file, persisted }) => ({ + file, + persisted, + })); + + try { + await onSendMessage(text, parts, attachmentsForMessage); + } catch (error) { + console.error("[chat] failed to send message", error); + const message = + error instanceof Error ? error.message : "Failed to send message"; + setAttachmentError(message); return; } - onSendMessage(text, [{ type: "text", text }]); editorRef.current?.editor?.commands.clearContent(); - }, [disabled, onSendMessage]); + setAttachedFiles([]); + setAttachmentError(null); + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + }, [attachedFiles, disabled, modalitySupport, onSendMessage]); useEffect(() => { const editor = editorRef.current?.editor; @@ -48,9 +159,72 @@ export function ChatMessageInput({ }, [disabled]); const handleAttachFile = useCallback(() => { - console.log("Attach file clicked"); + fileInputRef.current?.click(); }, []); + const handleFileChange = useCallback( + async (event: ChangeEvent) => { + const files = event.target.files; + if (files && files.length > 0) { + const validFiles: Array<{ + file: File; + persisted?: PersistedChatAttachment; + }> = []; + let errorMessage: string | null = null; + + for (const file of Array.from(files)) { + if (file.size > ATTACHMENT_SIZE_LIMIT) { + errorMessage = `File "${file.name}" exceeds ${Math.round(ATTACHMENT_SIZE_LIMIT / 1024 / 1024)}MB limit`; + break; + } + + if (chatGroupId) { + try { + const persisted = await saveChatAttachment(chatGroupId, file); + validFiles.push({ file, persisted }); + } catch (error) { + errorMessage = + error instanceof Error ? error.message : String(error); + break; + } + } else { + validFiles.push({ file }); + } + } + + if (errorMessage) { + setAttachmentError(errorMessage); + } else { + setAttachmentError(null); + setAttachedFiles((prev) => [...prev, ...validFiles]); + } + + if (fileInputRef.current) { + fileInputRef.current.value = ""; + } + } + }, + [chatGroupId], + ); + + const handleRemoveFile = useCallback( + (index: number) => { + setAttachedFiles((prev) => { + const toRemove = prev[index]; + if (toRemove?.persisted && chatGroupId) { + void removeChatAttachment(chatGroupId, toRemove.persisted.id).catch( + (error) => { + console.error("[chat] failed to remove attachment", error); + }, + ); + } + return prev.filter((_, i) => i !== index); + }); + setAttachmentError(null); + }, + [chatGroupId], + ); + const handleTakeScreenshot = useCallback(() => { console.log("Take screenshot clicked"); }, []); @@ -62,6 +236,49 @@ export function ChatMessageInput({ return (
+ + + {attachedFiles.length > 0 && ( +
+
+ {attachedFiles.map((item, index) => ( +
+ + {item.file.name} + + +
+ ))} +
+ {attachmentError && ( +
+ {attachmentError} +
+ )} + {capabilityWarning && ( +
+ {capabilityWarning} +
+ )} +
+ )} +
; } + if (part.type === "file") { + return } />; + } + if (part.type === "data-chat-file") { + return ( + } + /> + ); + } if (part.type === "step-start") { return null; } @@ -138,3 +148,67 @@ function Text({ part }: { part: Extract }) { ); } + +function FileAttachment({ part }: { part: Extract }) { + const isImage = part.mediaType?.startsWith("image/"); + const label = part.filename ?? "Attachment"; + + return ( + + ); +} + +function ChatFileAttachment({ + part, +}: { + part: Extract; +}) { + const isImage = part.data.mediaType?.startsWith("image/"); + const label = part.data.filename ?? "Attachment"; + + return ( + + ); +} diff --git a/apps/desktop/src/components/chat/message/types.ts b/apps/desktop/src/components/chat/message/types.ts index edf1f5b396..bccc53c682 100644 --- a/apps/desktop/src/components/chat/message/types.ts +++ b/apps/desktop/src/components/chat/message/types.ts @@ -2,8 +2,9 @@ import type { UIMessagePart } from "ai"; import type { ReactNode } from "react"; import type { ToolPartType, Tools } from "../../../chat/tools"; +import type { ChatDataParts } from "../../../chat/types"; -export type Part = UIMessagePart<{}, Tools>; +export type Part = UIMessagePart; export type ToolRenderer = ({ part, }: { diff --git a/apps/desktop/src/components/chat/session.tsx b/apps/desktop/src/components/chat/session.tsx index d3a8397972..1d04bd59a1 100644 --- a/apps/desktop/src/components/chat/session.tsx +++ b/apps/desktop/src/components/chat/session.tsx @@ -27,7 +27,7 @@ export function ChatSession({ chatGroupId, children, }: ChatSessionProps) { - const transport = useTransport(); + const transport = useTransport(chatGroupId); const store = main.UI.useStore(main.STORE_ID); const { user_id } = main.UI.useValues(main.STORE_ID); @@ -159,7 +159,7 @@ export function ChatSession({ ); } -function useTransport() { +function useTransport(chatGroupId?: string) { const registry = useToolRegistry(); const model = useLanguageModel(); @@ -168,8 +168,8 @@ function useTransport() { return null; } - return new CustomChatTransport(registry, model); - }, [registry, model]); + return new CustomChatTransport(registry, model, chatGroupId); + }, [registry, model, chatGroupId]); return transport; } diff --git a/apps/desktop/src/components/chat/view.tsx b/apps/desktop/src/components/chat/view.tsx index f9a10d6a15..2322ba893c 100644 --- a/apps/desktop/src/components/chat/view.tsx +++ b/apps/desktop/src/components/chat/view.tsx @@ -1,3 +1,4 @@ +import type { FileUIPart, TextUIPart } from "ai"; import { useCallback, useRef } from "react"; import type { HyprUIMessage } from "../../chat/types"; @@ -5,11 +6,17 @@ import { useShell } from "../../contexts/shell"; import { useLanguageModel } from "../../hooks/useLLMConnection"; import * as main from "../../store/tinybase/main"; import { id } from "../../utils"; +import { + type PersistedChatAttachment, + saveChatAttachment, +} from "./attachments/storage"; import { ChatBody } from "./body"; import { ChatHeader } from "./header"; import { ChatMessageInput } from "./input"; import { ChatSession } from "./session"; +type MessagePart = TextUIPart | FileUIPart | HyprUIMessage["parts"][number]; + export function ChatView() { const { chat } = useShell(); const { groupId, setGroupId } = chat; @@ -38,16 +45,16 @@ export function ChatView() { chat_group_id: string; content: string; role: string; - parts: any; - metadata: any; + parts: MessagePart[]; + metadata: { createdAt: number }; }) => p.id, (p: { id: string; chat_group_id: string; content: string; role: string; - parts: any; - metadata: any; + parts: MessagePart[]; + metadata: { createdAt: number }; }) => ({ user_id, chat_group_id: p.chat_group_id, @@ -62,34 +69,45 @@ export function ChatView() { ); const handleSendMessage = useCallback( - ( + async ( content: string, - parts: any[], + parts: MessagePart[], + attachments: Array<{ + file: File; + persisted?: PersistedChatAttachment; + }>, sendMessage: (message: HyprUIMessage) => void, ) => { const messageId = id(); - const uiMessage: HyprUIMessage = { - id: messageId, - role: "user", - parts, - metadata: { createdAt: Date.now() }, - }; - let currentGroupId = groupId; if (!currentGroupId) { currentGroupId = id(); - const title = content.slice(0, 50) + (content.length > 50 ? "..." : ""); + const title = deriveChatTitle(content, parts); createChatGroup({ groupId: currentGroupId, title }); setGroupId(currentGroupId); } + const normalizedParts = await ensurePersistedAttachmentParts( + parts, + attachments, + currentGroupId, + ); + + const metadata = { createdAt: Date.now() }; + const uiMessage: HyprUIMessage = { + id: messageId, + role: "user", + parts: normalizedParts, + metadata, + }; + createChatMessage({ id: messageId, chat_group_id: currentGroupId, content, role: "user", - parts, - metadata: { createdAt: Date.now() }, + parts: normalizedParts, + metadata, }); sendMessage(uiMessage); @@ -134,8 +152,9 @@ export function ChatView() { /> - handleSendMessage(content, parts, sendMessage) + chatGroupId={groupId} + onSendMessage={(content, parts, attachments) => + handleSendMessage(content, parts, attachments, sendMessage) } /> @@ -164,3 +183,79 @@ function useStableSessionId(groupId: string | undefined) { return sessionIdRef.current; } + +async function ensurePersistedAttachmentParts( + parts: MessagePart[], + attachments: Array<{ + file: File; + persisted?: PersistedChatAttachment; + }>, + chatGroupId: string, +): Promise { + const needsPersistence = attachments.some( + (attachment) => !attachment.persisted, + ); + + if (!needsPersistence) { + return parts; + } + + const newlySaved: PersistedChatAttachment[] = []; + + for (const attachment of attachments) { + if (attachment.persisted) { + continue; + } + const saved = await saveChatAttachment(chatGroupId, attachment.file); + newlySaved.push(saved); + } + + if (newlySaved.length === 0) { + return parts; + } + + let savedIndex = 0; + return parts.map((part) => { + if (part.type !== "file") { + return part; + } + + const saved = newlySaved[savedIndex++]; + if (!saved) { + return part; + } + + return { + type: "data-chat-file", + data: { + type: "chat-file", + attachmentId: saved.id, + filename: saved.fileName, + mediaType: saved.mimeType, + size: saved.size, + fileUrl: saved.fileUrl, + }, + }; + }); +} + +function deriveChatTitle(content: string, parts: MessagePart[]): string { + const fallback = "New chat"; + const trimmedContent = content.trim(); + const filePart = parts.find( + (part) => part.type === "file" || part.type === "data-chat-file", + ); + const filename = + filePart?.type === "file" + ? filePart.filename + : filePart?.type === "data-chat-file" + ? filePart.data.filename + : undefined; + const baseTitle = trimmedContent || filename || fallback; + + if (baseTitle.length <= 50) { + return baseTitle; + } + + return `${baseTitle.slice(0, 50)}...`; +} diff --git a/apps/desktop/src/components/main/body/sessions/note-input/attachments/index.tsx b/apps/desktop/src/components/main/body/sessions/note-input/attachments/index.tsx new file mode 100644 index 0000000000..fbe2009d85 --- /dev/null +++ b/apps/desktop/src/components/main/body/sessions/note-input/attachments/index.tsx @@ -0,0 +1,139 @@ +import { ImageIcon, LinkIcon, X } from "lucide-react"; + +import { formatDistanceToNow } from "@hypr/utils"; + +import type { Attachment } from "../index"; + +function AttachmentCard({ + attachment, + onRemove, +}: { + attachment: Attachment; + onRemove?: (id: string) => void; +}) { + const addedLabel = formatAttachmentTimestamp(attachment.addedAt); + + if (attachment.type === "link") { + return ( +
+ {onRemove && ( + + )} +
+
+ +
+
+
+ {attachment.title} +
+
{addedLabel}
+
+
+ {attachment.url && ( + + {attachment.url} + + )} +
+ ); + } + + return ( +
+ {onRemove && ( + + )} +
+ {attachment.thumbnailUrl ? ( + {attachment.title} + ) : ( +
+ +
+ )} + {!attachment.isPersisted && ( +
+ + Saving... + +
+ )} +
+
+
+ {attachment.title} +
+
{addedLabel}
+
+
+ ); +} + +export function Attachments({ + attachments, + onRemoveAttachment, + isLoading = false, +}: { + attachments: Attachment[]; + onRemoveAttachment?: (id: string) => void; + isLoading?: boolean; +}) { + return ( +
+
+ {isLoading ? ( +
+ Loading attachments... +
+ ) : attachments.length === 0 ? ( +
+ +

No attachments yet. Use the + icon above to add one.

+
+ ) : ( +
+ {attachments.map((attachment) => ( + + ))} +
+ )} +
+
+ ); +} + +function formatAttachmentTimestamp(value: string) { + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return value; + } + return formatDistanceToNow(date, { addSuffix: true }); +} diff --git a/apps/desktop/src/components/main/body/sessions/note-input/attachments/storage.ts b/apps/desktop/src/components/main/body/sessions/note-input/attachments/storage.ts new file mode 100644 index 0000000000..6955c7c672 --- /dev/null +++ b/apps/desktop/src/components/main/body/sessions/note-input/attachments/storage.ts @@ -0,0 +1,49 @@ +import { ATTACHMENT_SIZE_LIMIT } from "../../../../../../shared/attachments/constants"; +import { + createAttachmentStorage, + ManifestCorruptionError, + type ManifestEntry, +} from "../../../../../../shared/attachments/storage"; + +export type PersistedAttachment = ManifestEntry & { + title: string; + filePath: string; + fileUrl: string; +}; + +export { ManifestCorruptionError }; + +const sessionStorage = createAttachmentStorage< + ManifestEntry & { title: string } +>({ + getBasePath: (sessionId: string) => `hyprnote/sessions/${sessionId}`, + entityName: "session", + maxSize: ATTACHMENT_SIZE_LIMIT, + includeTitle: true, +}); + +export async function loadSessionAttachments( + sessionId: string, +): Promise { + return await sessionStorage.load(sessionId); +} + +export async function saveSessionAttachment( + sessionId: string, + file: File, + attachmentId = crypto.randomUUID(), +): Promise { + return await sessionStorage.save( + sessionId, + file, + { title: file.name || file.name || "attachment" }, + attachmentId, + ); +} + +export async function removeSessionAttachment( + sessionId: string, + attachmentId: string, +) { + return await sessionStorage.remove(sessionId, attachmentId); +} diff --git a/apps/desktop/src/components/main/body/sessions/note-input/enhanced/editor.tsx b/apps/desktop/src/components/main/body/sessions/note-input/enhanced/editor.tsx index abb782a323..6491af8e60 100644 --- a/apps/desktop/src/components/main/body/sessions/note-input/enhanced/editor.tsx +++ b/apps/desktop/src/components/main/body/sessions/note-input/enhanced/editor.tsx @@ -1,4 +1,4 @@ -import { forwardRef, useEffect, useMemo, useState } from "react"; +import { forwardRef, useCallback, useEffect, useMemo, useState } from "react"; import { type JSONContent, TiptapEditor } from "@hypr/tiptap/editor"; import NoteEditor from "@hypr/tiptap/editor"; @@ -8,8 +8,12 @@ import * as main from "../../../../../../store/tinybase/main"; export const EnhancedEditor = forwardRef< { editor: TiptapEditor | null }, - { sessionId: string; enhancedNoteId: string } ->(({ enhancedNoteId }, ref) => { + { + sessionId: string; + enhancedNoteId: string; + onContentChange?: (content: JSONContent) => void; + } +>(({ enhancedNoteId, onContentChange }, ref) => { const store = main.UI.useStore(main.STORE_ID); const [initialContent, setInitialContent] = useState(EMPTY_TIPTAP_DOC); @@ -34,7 +38,7 @@ export const EnhancedEditor = forwardRef< } }, [store, enhancedNoteId]); - const handleChange = main.UI.useSetPartialRowCallback( + const saveContent = main.UI.useSetPartialRowCallback( "enhanced_notes", enhancedNoteId, (input: JSONContent) => ({ content: JSON.stringify(input) }), @@ -42,6 +46,14 @@ export const EnhancedEditor = forwardRef< main.STORE_ID, ); + const handleChange = useCallback( + (input: JSONContent) => { + saveContent(input); + onContentChange?.(input); + }, + [saveContent, onContentChange], + ); + const mentionConfig = useMemo( () => ({ trigger: "@", diff --git a/apps/desktop/src/components/main/body/sessions/note-input/enhanced/index.tsx b/apps/desktop/src/components/main/body/sessions/note-input/enhanced/index.tsx index 37d832553c..d8ba38ddf4 100644 --- a/apps/desktop/src/components/main/body/sessions/note-input/enhanced/index.tsx +++ b/apps/desktop/src/components/main/body/sessions/note-input/enhanced/index.tsx @@ -1,6 +1,6 @@ import { forwardRef } from "react"; -import { type TiptapEditor } from "@hypr/tiptap/editor"; +import { type JSONContent, type TiptapEditor } from "@hypr/tiptap/editor"; import { useAITaskTask } from "../../../../../../hooks/useAITaskTask"; import { useLLMConnectionStatus } from "../../../../../../hooks/useLLMConnection"; @@ -11,8 +11,12 @@ import { StreamingView } from "./streaming"; export const Enhanced = forwardRef< { editor: TiptapEditor | null }, - { sessionId: string; enhancedNoteId: string } ->(({ sessionId, enhancedNoteId }, ref) => { + { + sessionId: string; + enhancedNoteId: string; + onContentChange?: (content: JSONContent) => void; + } +>(({ sessionId, enhancedNoteId, onContentChange }, ref) => { const taskId = createTaskId(enhancedNoteId, "enhance"); const llmStatus = useLLMConnectionStatus(); const { status } = useAITaskTask(taskId, "enhance"); @@ -40,6 +44,7 @@ export const Enhanced = forwardRef< ref={ref} sessionId={sessionId} enhancedNoteId={enhancedNoteId} + onContentChange={onContentChange} /> ); }); diff --git a/apps/desktop/src/components/main/body/sessions/note-input/header.tsx b/apps/desktop/src/components/main/body/sessions/note-input/header.tsx index 18be90e983..faf2ffebc0 100644 --- a/apps/desktop/src/components/main/body/sessions/note-input/header.tsx +++ b/apps/desktop/src/components/main/body/sessions/note-input/header.tsx @@ -1,4 +1,9 @@ -import { AlertCircleIcon, PlusIcon, RefreshCcwIcon } from "lucide-react"; +import { + AlertCircleIcon, + PlusIcon, + RefreshCcwIcon, + UploadIcon, +} from "lucide-react"; import { useCallback, useEffect, useRef, useState } from "react"; import { commands as windowsCommands } from "@hypr/plugin-windows"; @@ -57,6 +62,40 @@ function HeaderTab({ ); } +function HeaderTabAttachments({ + isActive, + onClick = () => {}, + onUploadClick, +}: { + isActive: boolean; + onClick?: () => void; + onUploadClick?: () => void; +}) { + const handleUploadClick = useCallback( + (event: React.MouseEvent) => { + event.stopPropagation(); + onUploadClick?.(); + }, + [onUploadClick], + ); + + return ( + + + Attachments + {isActive && onUploadClick && ( + + + + )} + + + ); +} + function TruncatedTitle({ title, isActive, @@ -301,6 +340,7 @@ export function Header({ handleTabChange, isEditing, setIsEditing, + onUploadAttachments, }: { sessionId: string; editorTabs: EditorView[]; @@ -308,10 +348,28 @@ export function Header({ handleTabChange: (view: EditorView) => void; isEditing: boolean; setIsEditing: (isEditing: boolean) => void; + onUploadAttachments?: (files: File[]) => void; }) { const sessionMode = useListener((state) => state.getSessionMode(sessionId)); const isBatchProcessing = sessionMode === "running_batch"; const isLiveProcessing = sessionMode === "running_active"; + const attachmentInputRef = useRef(null); + + const handleAttachmentUploadClick = useCallback(() => { + attachmentInputRef.current?.click(); + }, []); + + const handleAttachmentInputChange = useCallback( + (event: React.ChangeEvent) => { + const files = event.target.files; + if (!files || files.length === 0) { + return; + } + onUploadAttachments?.(Array.from(files)); + event.target.value = ""; + }, + [onUploadAttachments], + ); if (editorTabs.length === 1 && editorTabs[0].type === "raw") { return null; @@ -341,6 +399,21 @@ export function Header({ ); } + if (view.type === "attachments") { + return ( + handleTabChange(view)} + onUploadClick={ + onUploadAttachments + ? handleAttachmentUploadClick + : undefined + } + /> + ); + } + return ( ); })} - + {currentTab.type === "enhanced" && ( + + )}
{showProgress && } {showEditingControls && ( @@ -365,14 +440,25 @@ export function Header({ /> )}
+ ); } export function useEditorTabs({ sessionId, + shouldShowAttachments, }: { sessionId: string; + shouldShowAttachments: boolean; }): EditorView[] { const sessionMode = useListener((state) => state.getSessionMode(sessionId)); const hasTranscript = useHasTranscript(sessionId); @@ -381,9 +467,12 @@ export function useEditorTabs({ sessionId, main.STORE_ID, ); + const attachmentTabs: EditorView[] = shouldShowAttachments + ? [{ type: "attachments" }] + : []; if (sessionMode === "running_active" || sessionMode === "running_batch") { - return [{ type: "raw" }, { type: "transcript" }]; + return [{ type: "raw" }, { type: "transcript" }, ...attachmentTabs]; } if (hasTranscript) { @@ -391,10 +480,15 @@ export function useEditorTabs({ type: "enhanced", id, })); - return [...enhancedTabs, { type: "raw" }, { type: "transcript" }]; + return [ + ...enhancedTabs, + { type: "raw" }, + { type: "transcript" }, + ...attachmentTabs, + ]; } - return [{ type: "raw" }]; + return [{ type: "raw" }, ...attachmentTabs]; } function labelForEditorView(view: EditorView): string { @@ -407,6 +501,9 @@ function labelForEditorView(view: EditorView): string { if (view.type === "transcript") { return "Transcript"; } + if (view.type === "attachments") { + return "Attachments"; + } return ""; } diff --git a/apps/desktop/src/components/main/body/sessions/note-input/index.tsx b/apps/desktop/src/components/main/body/sessions/note-input/index.tsx index f07638255d..5bc405e3de 100644 --- a/apps/desktop/src/components/main/body/sessions/note-input/index.tsx +++ b/apps/desktop/src/components/main/body/sessions/note-input/index.tsx @@ -1,36 +1,102 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useHotkeys } from "react-hotkeys-hook"; -import type { TiptapEditor } from "@hypr/tiptap/editor"; +import type { JSONContent, TiptapEditor } from "@hypr/tiptap/editor"; +import { EMPTY_TIPTAP_DOC, isValidTiptapContent } from "@hypr/tiptap/shared"; import { cn } from "@hypr/utils"; import { useAutoEnhance } from "../../../../../hooks/useAutoEnhance"; import { useAutoTitle } from "../../../../../hooks/useAutoTitle"; +import * as main from "../../../../../store/tinybase/main"; import { type Tab, useTabs } from "../../../../../store/zustand/tabs"; import { type EditorView } from "../../../../../store/zustand/tabs/schema"; import { useCurrentNoteTab } from "../shared"; +import { Attachments } from "./attachments"; +import { + loadSessionAttachments, + ManifestCorruptionError, + type PersistedAttachment, + removeSessionAttachment, + saveSessionAttachment, +} from "./attachments/storage"; import { Enhanced } from "./enhanced"; import { Header, useEditorTabs } from "./header"; import { RawEditor } from "./raw"; import { Transcript } from "./transcript"; +export type Attachment = { + id: string; + type: "image" | "screenshot" | "link"; + title: string; + addedAt: string; + url?: string; + thumbnailUrl?: string; + objectUrl?: string; + fileUrl?: string; + mimeType?: string; + size?: number; + isPersisted?: boolean; +}; + +type AttachmentInsertOptions = { + position?: number; +}; + +type AttachmentInsertionPayload = { + id: string; + src: string; + position?: number; +}; + +type AttachmentSrcUpdatePayload = { + id: string; + src: string; +}; + export function NoteInput({ tab, }: { tab: Extract; }) { - const editorTabs = useEditorTabs({ sessionId: tab.id }); - const updateSessionTabState = useTabs((state) => state.updateSessionTabState); - const editorRef = useRef<{ editor: TiptapEditor | null }>(null); + const sessionId = tab.id; const [isEditing, setIsEditing] = useState(false); + const updateSessionTabState = useTabs((state) => state.updateSessionTabState); + const editorRef = useRef<{ editor: TiptapEditor | null } | null>(null); - const sessionId = tab.id; useAutoEnhance(tab); useAutoTitle(tab); const tabRef = useRef(tab); tabRef.current = tab; + const store = main.UI.useStore(main.STORE_ID); + const enhancedNoteIds = main.UI.useSliceRowIds( + main.INDEXES.enhancedNotesBySession, + sessionId, + main.STORE_ID, + ); + const enhancedNoteIdsRef = useRef([]); + useEffect(() => { + enhancedNoteIdsRef.current = enhancedNoteIds ?? []; + }, [enhancedNoteIds]); + + const { + attachments, + attachmentsLoading, + shouldShowAttachmentsTab, + attachmentsRef, + pendingAttachmentInsertionsRef, + pendingAttachmentSrcUpdatesRef, + pendingAttachmentSaves, + handleFilesAdded: sessionHandleFilesAdded, + handleRemoveAttachment, + } = useSessionAttachments(sessionId); + + const editorTabs = useEditorTabs({ + sessionId, + shouldShowAttachments: shouldShowAttachmentsTab, + }); + const handleTabChange = useCallback( (view: EditorView) => { updateSessionTabState(tabRef.current, { editor: view }); @@ -40,24 +106,374 @@ export function NoteInput({ const currentTab: EditorView = useCurrentNoteTab(tab); + useEffect(() => { + if (currentTab.type !== "attachments") { + return; + } + if (shouldShowAttachmentsTab) { + return; + } + const rawTab = editorTabs.find((view) => view.type === "raw"); + if (rawTab) { + handleTabChange(rawTab); + return; + } + if (editorTabs[0]) { + handleTabChange(editorTabs[0]); + } + }, [currentTab, shouldShowAttachmentsTab, editorTabs, handleTabChange]); + useTabShortcuts({ editorTabs, currentTab, handleTabChange, }); - useEffect(() => { - if (currentTab.type === "transcript" && editorRef.current) { - editorRef.current = { editor: null }; - } - }, [currentTab]); + const mountedEditorView = useMountedEditorView({ + currentTab, + editorTabs, + }); + + const performAttachmentInsertion = useCallback( + (editor: TiptapEditor, payload: AttachmentInsertionPayload) => { + const node = { + type: "image", + attrs: { + src: payload.src, + attachmentId: payload.id, + }, + }; + + if (typeof payload.position === "number") { + const inserted = editor + .chain() + .focus() + .insertContentAt(payload.position, node) + .run(); + if (inserted) { + return; + } + } + + editor.chain().focus().insertContent(node).run(); + }, + [], + ); + + const flushPendingAttachmentInsertions = useCallback( + (editor: TiptapEditor) => { + if (pendingAttachmentInsertionsRef.current.length === 0) { + return; + } + const pending = pendingAttachmentInsertionsRef.current; + pendingAttachmentInsertionsRef.current = []; + pending.forEach((payload) => performAttachmentInsertion(editor, payload)); + }, + [performAttachmentInsertion], + ); + + const insertAttachmentNode = useCallback( + (payload: AttachmentInsertionPayload) => { + const editor = editorRef.current?.editor; + if (!editor) { + pendingAttachmentInsertionsRef.current.push(payload); + return; + } + performAttachmentInsertion(editor, payload); + }, + [performAttachmentInsertion], + ); + + const performAttachmentSrcUpdate = useCallback( + (editor: TiptapEditor, payload: AttachmentSrcUpdatePayload) => { + editor.commands.command(({ tr, state, dispatch }) => { + let modified = false; + state.doc.descendants((node, pos) => { + if (node.type.name !== "image") { + return; + } + if (node.attrs.attachmentId === payload.id) { + tr.setNodeMarkup(pos, undefined, { + ...node.attrs, + src: payload.src, + }); + modified = true; + } + }); + if (!modified) { + return false; + } + if (dispatch) { + dispatch(tr); + } + return true; + }); + }, + [], + ); + + const flushPendingAttachmentSrcUpdates = useCallback( + (editor: TiptapEditor) => { + if (pendingAttachmentSrcUpdatesRef.current.length === 0) { + return; + } + const pending = pendingAttachmentSrcUpdatesRef.current; + pendingAttachmentSrcUpdatesRef.current = []; + pending.forEach((update) => performAttachmentSrcUpdate(editor, update)); + }, + [performAttachmentSrcUpdate], + ); + + const updateEditorAttachmentSrc = useCallback( + (update: AttachmentSrcUpdatePayload) => { + const editor = editorRef.current?.editor; + if (!editor) { + pendingAttachmentSrcUpdatesRef.current.push(update); + return; + } + performAttachmentSrcUpdate(editor, update); + }, + [performAttachmentSrcUpdate], + ); + + const removeAttachmentNodesFromEditor = useCallback( + (attachmentId: string, matchSrcs: string[] = []) => { + const editor = editorRef.current?.editor; + if (!editor) { + return; + } + editor.commands.command(({ tr, state, dispatch }) => { + const ranges: Array<{ from: number; to: number }> = []; + state.doc.descendants((node, pos) => { + if (node.type.name !== "image") { + return; + } + const matchesId = node.attrs.attachmentId === attachmentId; + const matchesSrc = + !node.attrs.attachmentId && + matchSrcs.some((src) => src && node.attrs.src === src); + if (matchesId || matchesSrc) { + ranges.push({ from: pos, to: pos + node.nodeSize }); + } + }); + if (ranges.length === 0) { + return false; + } + ranges + .sort((a, b) => b.from - a.from) + .forEach(({ from, to }) => { + tr.delete(from, to); + }); + if (dispatch) { + dispatch(tr); + } + return true; + }); + }, + [], + ); + + const handleFilesAdded = useCallback( + (files: File[], options?: AttachmentInsertOptions) => { + sessionHandleFilesAdded(files, { + ...options, + insertAttachmentNode, + updateEditorAttachmentSrc: updateEditorAttachmentSrc, + removeAttachmentNodesFromEditor, + }); + }, + [ + sessionHandleFilesAdded, + insertAttachmentNode, + removeAttachmentNodesFromEditor, + updateEditorAttachmentSrc, + ], + ); + + const handleEditorRef = useCallback( + (instance: { editor: TiptapEditor | null } | null) => { + editorRef.current = instance; + if (instance?.editor) { + flushPendingAttachmentInsertions(instance.editor); + flushPendingAttachmentSrcUpdates(instance.editor); + } + }, + [flushPendingAttachmentInsertions, flushPendingAttachmentSrcUpdates], + ); const handleContainerClick = () => { - if (currentTab.type !== "transcript") { - editorRef.current?.editor?.commands.focus(); + if (currentTab.type === "transcript" || currentTab.type === "attachments") { + return; } + editorRef.current?.editor?.commands.focus(); }; + const handleAttachmentUpload = useCallback( + (files: File[]) => { + handleFilesAdded(files); + }, + [handleFilesAdded], + ); + + const syncAttachmentsWithEditor = useCallback(() => { + const editor = editorRef.current?.editor; + if (!editor) { + return; + } + const referencedIds = new Set(); + const nodesMissingIds: Array<{ pos: number; attachmentId: string }> = []; + const nodesWithMissingAttachments: Array<{ from: number; to: number }> = []; + const attachmentIdSet = new Set( + attachmentsRef.current.map((attachment) => attachment.id), + ); + + const attachmentBySrc = new Map(); + attachmentsRef.current.forEach((attachment) => { + if (attachment.fileUrl) { + attachmentBySrc.set(attachment.fileUrl, attachment); + } + if (attachment.objectUrl) { + attachmentBySrc.set(attachment.objectUrl, attachment); + } + if (attachment.thumbnailUrl) { + attachmentBySrc.set(attachment.thumbnailUrl, attachment); + } + }); + + editor.state.doc.descendants((node, pos) => { + if (node.type.name !== "image") { + return; + } + let attachmentId = node.attrs.attachmentId as string | undefined; + + if (!attachmentId && node.attrs.src) { + const match = attachmentBySrc.get(node.attrs.src); + if (match) { + attachmentId = match.id; + nodesMissingIds.push({ pos, attachmentId }); + } + } + + if (attachmentId) { + referencedIds.add(attachmentId); + if (!attachmentIdSet.has(attachmentId)) { + nodesWithMissingAttachments.push({ + from: pos, + to: pos + node.nodeSize, + }); + } + } + }); + + if (nodesMissingIds.length > 0) { + editor.commands.command(({ tr, state, dispatch }) => { + let modified = false; + nodesMissingIds.forEach(({ pos, attachmentId }) => { + const node = state.doc.nodeAt(pos); + if (!node) { + return; + } + tr.setNodeMarkup(pos, undefined, { + ...node.attrs, + attachmentId, + }); + modified = true; + }); + if (!modified) { + return false; + } + if (dispatch) { + dispatch(tr); + } + return true; + }); + } + if (nodesWithMissingAttachments.length > 0) { + editor.commands.command(({ tr, dispatch }) => { + nodesWithMissingAttachments + .sort((a, b) => b.from - a.from) + .forEach(({ from, to }) => { + tr.delete(from, to); + }); + if (dispatch) { + dispatch(tr); + } + return true; + }); + } + }, []); + + const handleEditorContentChange = useCallback( + ( + content: JSONContent, + sourceView?: { type: "raw" } | { type: "enhanced"; id: string }, + ) => { + const { ids: referencedIds, srcs: referencedSrcs } = + collectAttachmentRefs(content); + + if ([...referencedSrcs].some((src) => src.startsWith("data:"))) { + console.log( + "[attachments] detected data URL references; skipping prune", + ); + return; + } + + const allReferencedIds = new Set(referencedIds); + const allReferencedSrcs = new Set(referencedSrcs); + + collectReferencesFromAllViews( + store, + sessionId, + sourceView, + enhancedNoteIdsRef.current, + allReferencedIds, + allReferencedSrcs, + ); + + console.log("[attachments] handleEditorContentChange", { + referencedIds: Array.from(allReferencedIds), + referencedSrcs: Array.from(allReferencedSrcs), + currentAttachments: attachmentsRef.current.map((a) => ({ + id: a.id, + isPersisted: a.isPersisted, + pending: pendingAttachmentSaves.current.has(a.id), + })), + }); + + const attachmentsToRemove = findOrphanedAttachments( + attachmentsRef.current, + allReferencedIds, + allReferencedSrcs, + pendingAttachmentSaves.current, + ); + + if (attachmentsToRemove.length > 0) { + console.log( + "[attachments] removing orphaned attachments", + attachmentsToRemove.map((a) => a.id), + ); + attachmentsToRemove.forEach((attachment) => { + handleRemoveAttachment(attachment.id, { skipStoredViewsPrune: true }); + }); + } + + syncAttachmentsWithEditor(); + }, + [handleRemoveAttachment, syncAttachmentsWithEditor, sessionId, store], + ); + + useEffect(() => { + if (!attachmentsLoading) { + syncAttachmentsWithEditor(); + } + }, [attachmentsLoading, syncAttachmentsWithEditor]); + + useEffect(() => { + if (!attachmentsLoading) { + syncAttachmentsWithEditor(); + } + }, [attachments, attachmentsLoading, syncAttachmentsWithEditor]); + return (
@@ -68,36 +484,524 @@ export function NoteInput({ handleTabChange={handleTabChange} isEditing={isEditing} setIsEditing={setIsEditing} + onUploadAttachments={handleAttachmentUpload} />
- {currentTab.type === "enhanced" && ( - - )} - {currentTab.type === "raw" && ( - - )} + + {currentTab.type === "transcript" && ( )} + {currentTab.type === "attachments" && ( + + )}
); } +function cleanupObjectUrls(urls: Set) { + urls.forEach((url) => { + URL.revokeObjectURL(url); + }); + urls.clear(); +} + +function useSessionAttachments(sessionId: string) { + const store = main.UI.useStore(main.STORE_ID); + const indexes = main.UI.useIndexes(main.STORE_ID); + const [attachments, setAttachments] = useState([]); + const [attachmentsLoading, setAttachmentsLoading] = useState(true); + const shouldShowAttachmentsTab = + !attachmentsLoading && attachments.length > 0; + + const attachmentsRef = useRef([]); + attachmentsRef.current = attachments; + + const pendingAttachmentInsertionsRef = useRef( + [], + ); + const pendingAttachmentSrcUpdatesRef = useRef( + [], + ); + const createdAttachmentUrls = useRef(new Set()); + const pendingAttachmentSaves = useRef( + new Map< + string, + { + cancelled: boolean; + } + >(), + ); + + const sessionIdRef = useRef(sessionId); + sessionIdRef.current = sessionId; + + useEffect(() => { + let cancelled = false; + + setAttachments([]); + setAttachmentsLoading(true); + + loadSessionAttachments(sessionId) + .then((loaded) => { + if (cancelled) { + return; + } + const persisted = loaded.map((record) => + mapPersistedAttachment(record), + ); + setAttachments((prev) => { + const optimistic = prev.filter( + (attachment) => !attachment.isPersisted, + ); + const optimisticIds = new Set( + optimistic.map((attachment) => attachment.id), + ); + const mergedPersisted = persisted.filter( + (attachment) => !optimisticIds.has(attachment.id), + ); + return [...optimistic, ...mergedPersisted]; + }); + }) + .catch((error) => { + if (error instanceof ManifestCorruptionError) { + console.error( + "[attachments] CRITICAL: manifest corruption detected", + error, + ); + } else { + console.error("[attachments] failed to load", error); + } + if (!cancelled) { + setAttachments([]); + } + }) + .finally(() => { + if (!cancelled) { + setAttachmentsLoading(false); + } + }); + + return () => { + cancelled = true; + pendingAttachmentSaves.current.forEach((entry) => { + entry.cancelled = true; + }); + }; + }, [sessionId]); + + const releaseObjectUrl = useCallback((url: string | undefined) => { + if (!url) { + return; + } + URL.revokeObjectURL(url); + createdAttachmentUrls.current.delete(url); + }, []); + + const removeAttachmentNodesFromStoredViews = useCallback( + (attachment: Attachment) => { + if (!store || !indexes) { + return; + } + + const prune = (serialized: unknown) => { + if (typeof serialized !== "string" || !serialized.trim()) { + return null; + } + const parsed = safeParseJSON(serialized); + if (!parsed || !isValidTiptapContent(parsed)) { + return null; + } + const { removed, content } = pruneAttachmentFromJson( + parsed, + attachment, + ); + if (!removed) { + return null; + } + return JSON.stringify(content); + }; + + const rawMd = store.getCell("sessions", sessionId, "raw_md"); + const nextRaw = prune(rawMd); + if (nextRaw !== null) { + store.setPartialRow("sessions", sessionId, { raw_md: nextRaw }); + } + + const enhancedNoteIds = indexes.getSliceRowIds( + main.INDEXES.enhancedNotesBySession, + sessionId, + ); + + enhancedNoteIds.forEach((noteId) => { + const content = store.getCell("enhanced_notes", noteId, "content"); + const next = prune(content); + if (next !== null) { + store.setPartialRow("enhanced_notes", noteId, { content: next }); + } + }); + }, + [indexes, sessionId, store], + ); + + const handleRemoveAttachment = useCallback( + (id: string, options?: { skipStoredViewsPrune?: boolean }) => { + console.log("[attachments] handleRemoveAttachment", { + id, + skipStoredViewsPrune: options?.skipStoredViewsPrune, + }); + + const attachmentToRemove = attachmentsRef.current.find( + (attachment) => attachment.id === id, + ); + if (attachmentToRemove?.objectUrl) { + releaseObjectUrl(attachmentToRemove.objectUrl); + } + + setAttachments((prev) => + prev.filter((attachment) => attachment.id !== id), + ); + + const pending = pendingAttachmentSaves.current.get(id); + if (pending) { + console.log("[attachments] cancelling pending save", id); + pending.cancelled = true; + } else { + console.log("[attachments] removing from storage", id); + void removeSessionAttachment(sessionId, id).catch((error) => { + console.error("[attachments] failed to remove", error); + }); + } + + if (!options?.skipStoredViewsPrune && attachmentToRemove) { + removeAttachmentNodesFromStoredViews(attachmentToRemove); + } + }, + [removeAttachmentNodesFromStoredViews, releaseObjectUrl, sessionId], + ); + + const handleFilesAdded = useCallback( + ( + files: File[], + options?: AttachmentInsertOptions & { + insertAttachmentNode?: (payload: AttachmentInsertionPayload) => void; + updateEditorAttachmentSrc?: ( + payload: AttachmentSrcUpdatePayload, + ) => void; + removeAttachmentNodesFromEditor?: ( + attachmentId: string, + matchSrcs?: string[], + ) => void; + }, + ) => { + let processedCount = 0; + const seenSignatures = new Set(); + + console.log("[attachments] handleFilesAdded", { + fileCount: files.length, + options, + }); + + const insertNode = + options?.insertAttachmentNode ?? + (() => { + /* no-op */ + }); + + const updateSrc = + options?.updateEditorAttachmentSrc ?? + (() => { + /* no-op */ + }); + + const removeFromEditor = + options?.removeAttachmentNodesFromEditor ?? + (() => { + /* no-op */ + }); + + files.forEach((file) => { + if (!file.type.startsWith("image/")) { + console.log("[attachments] skip non-image file", file.name); + return; + } + + const signature = `${file.name}-${file.size}-${file.lastModified}-${file.type}`; + if (seenSignatures.has(signature)) { + console.log("[attachments] skip duplicate file", file.name); + return; + } + seenSignatures.add(signature); + + const attachmentId = crypto.randomUUID(); + const objectUrl = URL.createObjectURL(file); + createdAttachmentUrls.current.add(objectUrl); + pendingAttachmentSaves.current.set(attachmentId, { cancelled: false }); + + console.log("[attachments] adding optimistic attachment", { + id: attachmentId, + fileName: file.name, + objectUrl, + }); + + const optimisticAttachment: Attachment = { + id: attachmentId, + type: "image", + title: file.name || "Image", + addedAt: new Date().toISOString(), + thumbnailUrl: objectUrl, + objectUrl, + mimeType: file.type, + size: file.size, + isPersisted: false, + }; + + setAttachments((prev) => [optimisticAttachment, ...prev]); + + insertNode({ + id: attachmentId, + src: objectUrl, + position: + processedCount === 0 ? (options?.position ?? undefined) : undefined, + }); + processedCount += 1; + + const sessionForSave = sessionId; + + void saveSessionAttachment(sessionForSave, file, attachmentId) + .then((saved) => { + const pending = pendingAttachmentSaves.current.get(attachmentId); + pendingAttachmentSaves.current.delete(attachmentId); + + console.log("[attachments] save complete", { + id: attachmentId, + savedId: saved.id, + sessionChanged: sessionIdRef.current !== sessionForSave, + wasCancelled: pending?.cancelled, + }); + + const sessionChanged = sessionIdRef.current !== sessionForSave; + const wasCancelled = pending?.cancelled; + + releaseObjectUrl(objectUrl); + + if (sessionChanged || wasCancelled) { + console.log( + "[attachments] discarding saved attachment", + saved.id, + ); + void removeSessionAttachment(sessionForSave, saved.id); + removeFromEditor(attachmentId, [objectUrl, saved.fileUrl]); + setAttachments((prev) => + prev.filter((attachment) => attachment.id !== attachmentId), + ); + return; + } + + setAttachments((prev) => { + const indexToUpdate = prev.findIndex( + (attachment) => attachment.id === attachmentId, + ); + if (indexToUpdate === -1) { + return prev; + } + const next = [...prev]; + next[indexToUpdate] = { + ...prev[indexToUpdate], + ...mapPersistedAttachment(saved), + objectUrl: undefined, + }; + return next; + }); + + updateSrc({ + id: attachmentId, + src: saved.fileUrl, + }); + }) + .catch((error) => { + pendingAttachmentSaves.current.delete(attachmentId); + console.error("[attachments] failed to save", error); + releaseObjectUrl(objectUrl); + removeFromEditor(attachmentId, [objectUrl]); + setAttachments((prev) => + prev.filter((attachment) => attachment.id !== attachmentId), + ); + }); + }); + }, + [releaseObjectUrl, sessionId], + ); + + useEffect(() => { + return () => { + cleanupObjectUrls(createdAttachmentUrls.current); + }; + }, [sessionId]); + + return { + attachments, + attachmentsLoading, + shouldShowAttachmentsTab, + attachmentsRef, + pendingAttachmentInsertionsRef, + pendingAttachmentSrcUpdatesRef, + pendingAttachmentSaves, + handleFilesAdded, + handleRemoveAttachment, + }; +} + +function mapPersistedAttachment(record: PersistedAttachment): Attachment { + return { + id: record.id, + type: "image", + title: record.title || record.fileName, + addedAt: record.addedAt, + thumbnailUrl: record.fileUrl, + fileUrl: record.fileUrl, + mimeType: record.mimeType, + size: record.size, + isPersisted: true, + }; +} + +function getAttachmentSrcCandidates(attachment: Attachment): string[] { + return [ + attachment.fileUrl, + attachment.objectUrl, + attachment.thumbnailUrl, + ].filter( + (value): value is string => typeof value === "string" && value.length > 0, + ); +} + +function collectAttachmentRefs(content: JSONContent | null | undefined) { + const ids = new Set(); + const srcs = new Set(); + + const visit = (node: JSONContent | undefined) => { + if (!node) { + return; + } + + if (node.type === "image") { + const nodeId = node.attrs?.attachmentId; + const nodeSrc = node.attrs?.src; + if (typeof nodeId === "string" && nodeId.length > 0) { + ids.add(nodeId); + } + if (typeof nodeSrc === "string" && nodeSrc.length > 0) { + srcs.add(nodeSrc); + } + } + + if (Array.isArray(node.content)) { + node.content.forEach((child) => { + visit(child as JSONContent); + }); + } + }; + + visit(content ?? undefined); + + return { ids, srcs }; +} + +function pruneAttachmentFromJson( + root: JSONContent, + attachment: Attachment, +): { removed: boolean; content: JSONContent } { + const srcCandidates = getAttachmentSrcCandidates(attachment); + let removed = false; + + const pruneNode = (node: JSONContent | undefined): JSONContent | null => { + if (!node) { + return null; + } + + if (node.type === "image") { + const nodeId = node.attrs?.attachmentId; + const nodeSrc = node.attrs?.src; + const matchesId = typeof nodeId === "string" && nodeId === attachment.id; + const matchesSrc = + (!nodeId || nodeId === "") && + typeof nodeSrc === "string" && + srcCandidates.includes(nodeSrc); + + if (matchesId || matchesSrc) { + removed = true; + return null; + } + } + + if (!Array.isArray(node.content)) { + return node; + } + + const nextContent: JSONContent[] = []; + let changed = false; + node.content.forEach((child) => { + const prunedChild = pruneNode(child as JSONContent); + if (prunedChild) { + nextContent.push(prunedChild); + if (prunedChild !== child) { + changed = true; + } + } else { + changed = true; + } + }); + + if (!changed) { + return node; + } + + return { ...node, content: nextContent }; + }; + + const pruned = pruneNode(root); + if (!removed) { + return { removed: false, content: root }; + } + + if (!pruned) { + return { removed: true, content: EMPTY_TIPTAP_DOC }; + } + + return { removed: true, content: pruned }; +} + +function safeParseJSON(value: string): JSONContent | null { + try { + return JSON.parse(value) as JSONContent; + } catch { + return null; + } +} + function useTabShortcuts({ editorTabs, currentTab, @@ -162,4 +1066,233 @@ function useTabShortcuts({ }, [currentTab, editorTabs, handleTabChange], ); + + useHotkeys( + "alt+a", + () => { + const attachmentsTab = editorTabs.find((t) => t.type === "attachments"); + if (attachmentsTab && currentTab.type !== "attachments") { + handleTabChange(attachmentsTab); + } + }, + { + preventDefault: true, + enableOnFormTags: true, + enableOnContentEditable: true, + }, + [currentTab, editorTabs, handleTabChange], + ); +} + +function EditorViewContainer({ + currentTab, + mountedEditorView, + sessionId, + handleEditorRef, + handleFilesAdded, + handleContentChange, +}: { + currentTab: EditorView; + mountedEditorView: MountedEditorView; + sessionId: string; + handleEditorRef: (instance: { editor: TiptapEditor | null } | null) => void; + handleFilesAdded: (files: File[], options?: AttachmentInsertOptions) => void; + handleContentChange: ( + content: JSONContent, + sourceView?: { type: "raw" } | { type: "enhanced"; id: string }, + ) => void; +}) { + const editorViewToRender = + currentTab.type === "raw" || currentTab.type === "enhanced" + ? currentTab + : mountedEditorView; + + const handleRawContentChange = useCallback( + (content: JSONContent) => { + handleContentChange(content, { type: "raw" }); + }, + [handleContentChange], + ); + + const handleEnhancedContentChange = useCallback( + (content: JSONContent) => { + if (editorViewToRender?.type === "enhanced") { + handleContentChange(content, { + type: "enhanced", + id: editorViewToRender.id, + }); + } + }, + [handleContentChange, editorViewToRender], + ); + + const allowFileDrop = editorViewToRender?.type === "raw"; + + return ( +
+ {editorViewToRender?.type === "raw" ? ( + + ) : editorViewToRender?.type === "enhanced" ? ( + + ) : null} +
+ ); +} + +type MountedEditorView = Extract; + +function useMountedEditorView({ + currentTab, + editorTabs, +}: { + currentTab: EditorView; + editorTabs: EditorView[]; +}): MountedEditorView { + const initialEditorView = useMemo(() => { + if (currentTab.type === "raw" || currentTab.type === "enhanced") { + return currentTab; + } + const rawTab = editorTabs.find( + (tab): tab is Extract => tab.type === "raw", + ); + if (rawTab) { + return rawTab; + } + const enhancedTab = editorTabs.find( + (tab): tab is Extract => + tab.type === "enhanced", + ); + if (enhancedTab) { + return enhancedTab; + } + return { type: "raw" }; + }, [currentTab, editorTabs]); + + const [mountedEditorView, setMountedEditorView] = + useState(initialEditorView); + + useEffect(() => { + if (currentTab.type === "raw" || currentTab.type === "enhanced") { + setMountedEditorView(currentTab); + } + }, [currentTab]); + + useEffect(() => { + if ( + mountedEditorView.type === "enhanced" && + !editorTabs.some( + (tab) => tab.type === "enhanced" && tab.id === mountedEditorView.id, + ) + ) { + const rawTab = editorTabs.find( + (tab): tab is Extract => + tab.type === "raw", + ); + if (rawTab) { + setMountedEditorView(rawTab); + return; + } + const enhancedTab = editorTabs.find( + (tab): tab is Extract => + tab.type === "enhanced", + ); + if (enhancedTab) { + setMountedEditorView(enhancedTab); + return; + } + setMountedEditorView({ type: "raw" }); + } + }, [editorTabs, mountedEditorView]); + + return mountedEditorView; +} + +function collectReferencesFromAllViews( + store: ReturnType | undefined, + sessionId: string, + sourceView: { type: "raw" } | { type: "enhanced"; id: string } | undefined, + enhancedNoteIds: string[], + allReferencedIds: Set, + allReferencedSrcs: Set, +) { + if (!store) { + return; + } + + if (!sourceView || sourceView.type !== "raw") { + const rawMd = store.getCell("sessions", sessionId, "raw_md"); + if (typeof rawMd === "string" && rawMd.trim()) { + const parsed = safeParseJSON(rawMd); + if (parsed && isValidTiptapContent(parsed)) { + const { ids, srcs } = collectAttachmentRefs(parsed); + ids.forEach((id) => allReferencedIds.add(id)); + srcs.forEach((src) => allReferencedSrcs.add(src)); + } + } + } + + enhancedNoteIds.forEach((noteId) => { + if (sourceView?.type === "enhanced" && sourceView.id === noteId) { + return; + } + const noteMd = store.getCell("enhanced_notes", noteId, "content"); + if (typeof noteMd === "string" && noteMd.trim()) { + const parsed = safeParseJSON(noteMd); + if (parsed && isValidTiptapContent(parsed)) { + const { ids, srcs } = collectAttachmentRefs(parsed); + ids.forEach((id) => allReferencedIds.add(id)); + srcs.forEach((src) => allReferencedSrcs.add(src)); + } + } + }); +} + +function findOrphanedAttachments( + attachments: Attachment[], + allReferencedIds: Set, + allReferencedSrcs: Set, + pendingAttachmentSaves: Map, +): Attachment[] { + return attachments.filter((attachment) => { + if (!attachment.isPersisted) { + console.log("[attachments] skip remove (not persisted)", attachment.id); + return false; + } + if (pendingAttachmentSaves.has(attachment.id)) { + console.log("[attachments] skip remove (pending save)", attachment.id); + return false; + } + if (allReferencedIds.has(attachment.id)) { + console.log("[attachments] keep (referenced by ID)", attachment.id); + return false; + } + const candidates = getAttachmentSrcCandidates(attachment); + const isReferencedBySrc = candidates.some((src) => + allReferencedSrcs.has(src), + ); + if (isReferencedBySrc) { + console.log("[attachments] keep (referenced by src)", attachment.id); + } else { + console.log("[attachments] REMOVE (not referenced)", attachment.id); + } + return !isReferencedBySrc; + }); } diff --git a/apps/desktop/src/components/main/body/sessions/note-input/raw.tsx b/apps/desktop/src/components/main/body/sessions/note-input/raw.tsx index 4099a98db9..23ceb35380 100644 --- a/apps/desktop/src/components/main/body/sessions/note-input/raw.tsx +++ b/apps/desktop/src/components/main/body/sessions/note-input/raw.tsx @@ -1,4 +1,4 @@ -import { forwardRef, useEffect, useMemo, useState } from "react"; +import { forwardRef, useCallback, useEffect, useMemo, useState } from "react"; import NoteEditor, { type JSONContent, @@ -6,6 +6,7 @@ import NoteEditor, { } from "@hypr/tiptap/editor"; import { EMPTY_TIPTAP_DOC, + type FileHandlerConfig, isValidTiptapContent, type PlaceholderFunction, } from "@hypr/tiptap/shared"; @@ -14,8 +15,12 @@ import * as main from "../../../../../store/tinybase/main"; export const RawEditor = forwardRef< { editor: TiptapEditor | null }, - { sessionId: string } ->(({ sessionId }, ref) => { + { + sessionId: string; + onFilesAdded?: (files: File[], options?: { position?: number }) => void; + onContentChange?: (content: JSONContent) => void; + } +>(({ sessionId, onFilesAdded, onContentChange }, ref) => { const store = main.UI.useStore(main.STORE_ID); const [initialContent, setInitialContent] = @@ -41,7 +46,7 @@ export const RawEditor = forwardRef< } }, [store, sessionId]); - const handleChange = main.UI.useSetPartialRowCallback( + const saveContent = main.UI.useSetPartialRowCallback( "sessions", sessionId, (input: JSONContent) => ({ raw_md: JSON.stringify(input) }), @@ -49,6 +54,14 @@ export const RawEditor = forwardRef< main.STORE_ID, ); + const handleChange = useCallback( + (input: JSONContent) => { + saveContent(input); + onContentChange?.(input); + }, + [saveContent, onContentChange], + ); + const mentionConfig = useMemo( () => ({ trigger: "@", @@ -59,6 +72,28 @@ export const RawEditor = forwardRef< [], ); + const fileHandlerConfig = useMemo( + () => + onFilesAdded + ? ({ + onDrop: ( + files: File[], + _editor: TiptapEditor, + position?: number, + ) => { + onFilesAdded(files, { position }); + return false; + }, + onPaste: (files: File[], editor: TiptapEditor) => { + const pos = editor.state.selection.from; + onFilesAdded(files, { position: pos }); + return false; + }, + } satisfies FileHandlerConfig) + : undefined, + [onFilesAdded], + ); + return ( ); }); diff --git a/apps/desktop/src/shared/attachments/constants.ts b/apps/desktop/src/shared/attachments/constants.ts new file mode 100644 index 0000000000..bea5b7fa28 --- /dev/null +++ b/apps/desktop/src/shared/attachments/constants.ts @@ -0,0 +1 @@ +export const ATTACHMENT_SIZE_LIMIT = 10 * 1024 * 1024; diff --git a/apps/desktop/src/shared/attachments/storage.ts b/apps/desktop/src/shared/attachments/storage.ts new file mode 100644 index 0000000000..1a458af061 --- /dev/null +++ b/apps/desktop/src/shared/attachments/storage.ts @@ -0,0 +1,396 @@ +import { convertFileSrc } from "@tauri-apps/api/core"; +import { dataDir, join } from "@tauri-apps/api/path"; +import { + BaseDirectory, + exists, + mkdir, + readFile, + readTextFile, + remove as removeFile, + writeFile, + writeTextFile, +} from "@tauri-apps/plugin-fs"; +import { Mutex } from "async-mutex"; + +export type ManifestEntry = { + id: string; + fileName: string; + mimeType: string; + addedAt: string; + size: number; + title?: string; +}; + +type Manifest = Record; + +export class ManifestCorruptionError extends Error { + constructor( + message: string, + public readonly cause?: unknown, + ) { + super(message); + this.name = "ManifestCorruptionError"; + } +} + +type MutexEntry = { + mutex: Mutex; + refCount: number; + lastUsed: number; +}; + +const manifestLocks = new Map(); +const MUTEX_CLEANUP_INTERVAL = 60_000; +const MUTEX_IDLE_THRESHOLD = 300_000; + +let cleanupTimer: ReturnType | null = null; + +function startMutexCleanup() { + if (cleanupTimer) return; + cleanupTimer = setInterval(() => { + const now = Date.now(); + const toDelete: string[] = []; + for (const [id, entry] of manifestLocks.entries()) { + if (entry.refCount === 0 && now - entry.lastUsed > MUTEX_IDLE_THRESHOLD) { + toDelete.push(id); + } + } + toDelete.forEach((id) => manifestLocks.delete(id)); + }, MUTEX_CLEANUP_INTERVAL); +} + +function getManifestMutex(id: string): Mutex { + startMutexCleanup(); + let entry = manifestLocks.get(id); + if (!entry) { + entry = { + mutex: new Mutex(), + refCount: 0, + lastUsed: Date.now(), + }; + manifestLocks.set(id, entry); + } + return entry.mutex; +} + +function acquireMutex(id: string) { + const entry = manifestLocks.get(id); + if (entry) { + entry.refCount++; + entry.lastUsed = Date.now(); + } +} + +function releaseMutex(id: string) { + const entry = manifestLocks.get(id); + if (entry) { + entry.refCount = Math.max(0, entry.refCount - 1); + entry.lastUsed = Date.now(); + } +} + +export interface StorageConfig { + getBasePath: (id: string) => string; + entityName: string; + maxSize: number; + includeTitle?: boolean; +} + +export function createAttachmentStorage( + config: StorageConfig, +) { + const BASE_DIR = config.getBasePath; + const ATTACHMENTS_DIR = (id: string) => `${BASE_DIR(id)}/attachments`; + const ATTACHMENTS_MANIFEST_PATH = (id: string) => + `${ATTACHMENTS_DIR(id)}/attachments.json`; + const ATTACHMENT_ENTRY_DIR = (id: string, attachmentId: string) => + `${ATTACHMENTS_DIR(id)}/${attachmentId}`; + const ATTACHMENT_FILE_PATH = ( + id: string, + attachmentId: string, + fileName: string, + ) => `${ATTACHMENT_ENTRY_DIR(id, attachmentId)}/${fileName}`; + + const DEFAULT_MANIFEST: Manifest = {}; + + async function ensureDirectory(path: string) { + const dirExists = await exists(path, { baseDir: BaseDirectory.Data }); + if (!dirExists) { + await mkdir(path, { + baseDir: BaseDirectory.Data, + recursive: true, + }); + } + } + + async function toAbsolutePath(relativePath: string) { + const resolvedDataDir = await dataDir(); + return await join(resolvedDataDir, relativePath); + } + + async function readManifest(id: string): Promise { + const manifestPath = ATTACHMENTS_MANIFEST_PATH(id); + const manifestExists = await exists(manifestPath, { + baseDir: BaseDirectory.Data, + }); + + if (!manifestExists) { + return { ...DEFAULT_MANIFEST }; + } + + try { + const content = await readTextFile(manifestPath, { + baseDir: BaseDirectory.Data, + }); + const parsed = JSON.parse(content) as Manifest; + + if ( + typeof parsed !== "object" || + parsed === null || + Array.isArray(parsed) + ) { + throw new ManifestCorruptionError( + `Invalid manifest structure: expected object, got ${typeof parsed}`, + ); + } + + for (const [key, value] of Object.entries(parsed)) { + const requiredFields = + !value || + typeof value !== "object" || + typeof value.id !== "string" || + typeof value.fileName !== "string" || + typeof value.mimeType !== "string" || + typeof value.addedAt !== "string" || + typeof value.size !== "number"; + + const titleCheck = + config.includeTitle && typeof value.title !== "string"; + + if (requiredFields || titleCheck) { + throw new ManifestCorruptionError( + `Invalid manifest entry for key ${key}: missing or invalid required fields`, + ); + } + } + + return parsed; + } catch (error) { + if (error instanceof ManifestCorruptionError) { + throw error; + } + throw new ManifestCorruptionError( + `Failed to read or parse manifest for ${config.entityName} ${id}`, + error, + ); + } + } + + async function writeManifest(id: string, manifest: Manifest) { + const manifestPath = ATTACHMENTS_MANIFEST_PATH(id); + await ensureDirectory(ATTACHMENTS_DIR(id)); + await writeTextFile(manifestPath, JSON.stringify(manifest), { + baseDir: BaseDirectory.Data, + }); + } + + function sanitizeFileName(name: string) { + const sanitized = name + .replace(/[<>:"/\\|?*\u0000-\u001F]/g, "_") + .slice(0, 255); + return sanitized.length > 0 ? sanitized : "attachment"; + } + + async function load( + id: string, + ): Promise> { + const mutex = getManifestMutex(id); + return await mutex.runExclusive(async () => { + acquireMutex(id); + try { + await ensureDirectory(ATTACHMENTS_DIR(id)); + const manifest = await readManifest(id); + const entries = Object.values(manifest); + + const results: Array = []; + + for (const entry of entries) { + const relativePath = ATTACHMENT_FILE_PATH( + id, + entry.id, + entry.fileName, + ); + const existsOnDisk = await exists(relativePath, { + baseDir: BaseDirectory.Data, + }); + if (!existsOnDisk) { + continue; + } + + const absolutePath = await toAbsolutePath(relativePath); + results.push({ + ...(entry as T), + filePath: absolutePath, + fileUrl: convertFileSrc(absolutePath), + }); + } + + return results.sort((a, b) => + a.addedAt > b.addedAt ? -1 : a.addedAt < b.addedAt ? 1 : 0, + ); + } finally { + releaseMutex(id); + } + }); + } + + async function save( + id: string, + file: File, + metadata: Partial, + attachmentId = crypto.randomUUID(), + ): Promise { + if (file.size > config.maxSize) { + throw new Error( + `Attachment size ${file.size} exceeds maximum allowed size of ${config.maxSize} bytes`, + ); + } + + const mutex = getManifestMutex(id); + return await mutex.runExclusive(async () => { + acquireMutex(id); + try { + const safeFileName = sanitizeFileName(file.name || "attachment"); + const relativeDir = ATTACHMENT_ENTRY_DIR(id, attachmentId); + + await ensureDirectory(relativeDir); + + const relativeFilePath = ATTACHMENT_FILE_PATH( + id, + attachmentId, + safeFileName, + ); + + const fileBuffer = await file.arrayBuffer(); + await writeFile(relativeFilePath, new Uint8Array(fileBuffer), { + baseDir: BaseDirectory.Data, + }); + + const manifest = await readManifest(id); + const entry: ManifestEntry = { + id: attachmentId, + fileName: safeFileName, + mimeType: file.type || "application/octet-stream", + addedAt: new Date().toISOString(), + size: file.size, + ...metadata, + }; + + manifest[attachmentId] = entry; + await writeManifest(id, manifest); + + const absolutePath = await toAbsolutePath(relativeFilePath); + + return { + ...(entry as T), + filePath: absolutePath, + fileUrl: convertFileSrc(absolutePath), + }; + } finally { + releaseMutex(id); + } + }); + } + + async function remove(id: string, attachmentId: string) { + const mutex = getManifestMutex(id); + return await mutex.runExclusive(async () => { + acquireMutex(id); + try { + const manifest = await readManifest(id); + + if (manifest[attachmentId]) { + delete manifest[attachmentId]; + await writeManifest(id, manifest); + } + + const attachmentDir = ATTACHMENT_ENTRY_DIR(id, attachmentId); + const dirExists = await exists(attachmentDir, { + baseDir: BaseDirectory.Data, + }); + + if (dirExists) { + await removeFile(attachmentDir, { + baseDir: BaseDirectory.Data, + recursive: true, + }); + } + } finally { + releaseMutex(id); + } + }); + } + + async function removeAll(id: string) { + const baseDir = BASE_DIR(id); + const dirExists = await exists(baseDir, { + baseDir: BaseDirectory.Data, + }); + + if (dirExists) { + await removeFile(baseDir, { + baseDir: BaseDirectory.Data, + recursive: true, + }); + } + } + + async function readAsDataURL( + id: string, + attachmentId: string, + ): Promise { + const mutex = getManifestMutex(id); + return await mutex.runExclusive(async () => { + acquireMutex(id); + try { + const manifest = await readManifest(id); + const entry = manifest[attachmentId]; + + if (!entry) { + return null; + } + + const relativePath = ATTACHMENT_FILE_PATH( + id, + attachmentId, + entry.fileName, + ); + const existsOnDisk = await exists(relativePath, { + baseDir: BaseDirectory.Data, + }); + + if (!existsOnDisk) { + return null; + } + + const fileData = await readFile(relativePath, { + baseDir: BaseDirectory.Data, + }); + + const base64 = btoa(String.fromCharCode(...new Uint8Array(fileData))); + + return `data:${entry.mimeType};base64,${base64}`; + } finally { + releaseMutex(id); + } + }); + } + + return { + load, + save, + remove, + removeAll, + readAsDataURL, + }; +} diff --git a/apps/desktop/src/store/zustand/ai-task/shared/resolve-attachments.ts b/apps/desktop/src/store/zustand/ai-task/shared/resolve-attachments.ts new file mode 100644 index 0000000000..536f10c922 --- /dev/null +++ b/apps/desktop/src/store/zustand/ai-task/shared/resolve-attachments.ts @@ -0,0 +1,98 @@ +import { readFile } from "@tauri-apps/plugin-fs"; +import { BaseDirectory, exists } from "@tauri-apps/plugin-fs"; +import type { FilePart } from "ai"; + +import { ATTACHMENT_SIZE_LIMIT } from "../../../../shared/attachments/constants"; + +type AttachmentReference = { + id: string; + fileName: string; + mimeType: string; + size: number; + fileUrl: string; +}; + +export async function resolveSessionAttachments( + sessionId: string, + attachments: AttachmentReference[], +): Promise> { + const resolved: Array = []; + + for (const attachment of attachments) { + if (attachment.size > ATTACHMENT_SIZE_LIMIT) { + console.warn( + `[resolve-attachments] skipping ${attachment.fileName}: exceeds ${ATTACHMENT_SIZE_LIMIT / 1024 / 1024}MB limit`, + ); + continue; + } + + if (!attachment.mimeType.startsWith("image/")) { + console.warn( + `[resolve-attachments] skipping ${attachment.fileName}: unsupported type ${attachment.mimeType}`, + ); + continue; + } + + try { + const dataUrl = await readAttachmentAsDataURL( + sessionId, + attachment.id, + attachment.fileName, + attachment.mimeType, + ); + + if (!dataUrl) { + console.warn( + `[resolve-attachments] skipping ${attachment.fileName}: failed to read`, + ); + continue; + } + + resolved.push({ + type: "file", + data: dataUrl, + mediaType: attachment.mimeType, + }); + } catch (error) { + console.error( + `[resolve-attachments] error reading ${attachment.fileName}:`, + error, + ); + } + } + + return resolved; +} + +async function readAttachmentAsDataURL( + sessionId: string, + attachmentId: string, + fileName: string, + mimeType: string, +): Promise { + const relativePath = `hyprnote/sessions/${sessionId}/attachments/${attachmentId}/${fileName}`; + + const existsOnDisk = await exists(relativePath, { + baseDir: BaseDirectory.Data, + }); + + if (!existsOnDisk) { + return null; + } + + try { + const fileData = await readFile(relativePath, { + baseDir: BaseDirectory.Data, + }); + + const base64 = btoa(String.fromCharCode(...new Uint8Array(fileData))); + + return `data:${mimeType};base64,${base64}`; + } catch (error) { + console.error( + `[resolve-attachments] failed to read ${relativePath}:`, + error, + ); + return null; + } +} diff --git a/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-transform.ts b/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-transform.ts index ce9f0500c4..757c9d02b4 100644 --- a/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-transform.ts +++ b/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-transform.ts @@ -1,4 +1,5 @@ import type { TaskArgsMap, TaskArgsMapTransformed, TaskConfig } from "."; +import { loadSessionAttachments } from "../../../../components/main/body/sessions/note-input/attachments/storage"; import { buildSegments, type RuntimeSpeakerHint, @@ -46,6 +47,7 @@ async function transformArgs( const sessionContext = getSessionContext(sessionId, store); const template = templateId ? getTemplateData(templateId, store) : undefined; + const attachments = await getSessionAttachments(sessionId); return { sessionId, @@ -55,6 +57,7 @@ async function transformArgs( participants: sessionContext.participants, segments: sessionContext.segments, template, + attachments, }; } @@ -377,3 +380,19 @@ function getNumberCell( const value = store.getCell(tableId, rowId, columnId); return typeof value === "number" ? value : undefined; } + +async function getSessionAttachments(sessionId: string) { + try { + const loaded = await loadSessionAttachments(sessionId); + return loaded.map((attachment) => ({ + id: attachment.id, + fileName: attachment.fileName, + mimeType: attachment.mimeType, + size: attachment.size, + fileUrl: attachment.fileUrl, + })); + } catch (error) { + console.error("[enhance-transform] failed to load attachments", error); + return []; + } +} diff --git a/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-workflow.ts b/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-workflow.ts index b32165cd91..04376c0720 100644 --- a/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-workflow.ts +++ b/apps/desktop/src/store/zustand/ai-task/task-configs/enhance-workflow.ts @@ -14,6 +14,7 @@ import { TemplateSection, templateSectionSchema, } from "../../../tinybase/schema-external"; +import { resolveSessionAttachments } from "../shared/resolve-attachments"; import { addMarkdownSectionSeparators, trimBeforeMarker, @@ -158,6 +159,11 @@ async function* generateSummary(params: { const validator = createValidator(args.template); + const attachmentParts = + args.attachments.length > 0 + ? await resolveSessionAttachments(args.sessionId, args.attachments) + : []; + yield* withEarlyValidationRetry( (retrySignal, { previousFeedback }) => { let enhancedPrompt = prompt; @@ -177,10 +183,20 @@ IMPORTANT: Previous attempt failed. ${previousFeedback}`; retrySignal.addEventListener("abort", abortFromRetry); try { + const messages: Array = [ + { + role: "user", + content: [ + { type: "text", text: enhancedPrompt }, + ...attachmentParts, + ], + }, + ]; + const result = streamText({ model, system, - prompt: enhancedPrompt, + messages, abortSignal: combinedController.signal, }); return result.fullStream; diff --git a/apps/desktop/src/store/zustand/ai-task/task-configs/index.ts b/apps/desktop/src/store/zustand/ai-task/task-configs/index.ts index a987b2fb12..fe02315bd8 100644 --- a/apps/desktop/src/store/zustand/ai-task/task-configs/index.ts +++ b/apps/desktop/src/store/zustand/ai-task/task-configs/index.ts @@ -44,6 +44,13 @@ export interface TaskArgsMapTransformed { }>; }>; template?: Pick; + attachments: Array<{ + id: string; + fileName: string; + mimeType: string; + size: number; + fileUrl: string; + }>; }; title: { sessionId: string; diff --git a/apps/desktop/src/store/zustand/tabs/schema.ts b/apps/desktop/src/store/zustand/tabs/schema.ts index 597c947b88..ad61ee87b4 100644 --- a/apps/desktop/src/store/zustand/tabs/schema.ts +++ b/apps/desktop/src/store/zustand/tabs/schema.ts @@ -10,6 +10,7 @@ const baseTabSchema = z.object({ export const editorViewSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal("raw") }), z.object({ type: z.literal("transcript") }), + z.object({ type: z.literal("attachments") }), z.object({ type: z.literal("enhanced"), id: z.string(), @@ -25,6 +26,9 @@ export const isRawView = (view: EditorView): view is { type: "raw" } => export const isTranscriptView = ( view: EditorView, ): view is { type: "transcript" } => view.type === "transcript"; +export const isAttachmentsView = ( + view: EditorView, +): view is { type: "attachments" } => view.type === "attachments"; export const tabSchema = z.discriminatedUnion("type", [ baseTabSchema.extend({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 541383fab2..8eff26199e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -234,6 +234,9 @@ importers: ai: specifier: ^5.0.93 version: 5.0.93(zod@4.1.12) + async-mutex: + specifier: ^0.5.0 + version: 0.5.0 chroma-js: specifier: ^3.1.2 version: 3.1.2 @@ -333,10 +336,10 @@ importers: version: 10.1.0 '@tanstack/react-router-devtools': specifier: ^1.136.8 - version: 1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1) + version: 1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@1.21.7)(lightningcss@1.30.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1) '@tanstack/router-plugin': specifier: ^1.136.8 - version: 1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1)) + version: 1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(vite@7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1)) '@tauri-apps/cli': specifier: ^2.9.4 version: 2.9.4 @@ -360,7 +363,7 @@ importers: version: 2.0.3 '@vitejs/plugin-react': specifier: ^4.7.0 - version: 4.7.0(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1)) + version: 4.7.0(vite@7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1)) autoprefixer: specifier: ^10.4.22 version: 10.4.22(postcss@8.5.6) @@ -384,10 +387,10 @@ importers: version: 5.8.3 vite: specifier: ^7.2.2 - version: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + version: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) vitest: specifier: ^3.2.4 - version: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.1)(jiti@2.6.1)(jsdom@27.2.0)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + version: 3.2.4(@types/debug@4.1.12)(@types/node@24.10.1)(jiti@1.21.7)(jsdom@27.2.0)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) apps/pro: dependencies: @@ -5908,6 +5911,9 @@ packages: resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} hasBin: true + async-mutex@0.5.0: + resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==} + async-sema@3.1.1: resolution: {integrity: sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==} @@ -16826,13 +16832,13 @@ snapshots: - tsx - yaml - '@tanstack/react-router-devtools@1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1)': + '@tanstack/react-router-devtools@1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@1.21.7)(lightningcss@1.30.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1)': dependencies: '@tanstack/react-router': 1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0) - '@tanstack/router-devtools-core': 1.136.8(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1) + '@tanstack/router-devtools-core': 1.136.8(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@1.21.7)(lightningcss@1.30.2)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1) react: 19.2.0 react-dom: 19.2.0(react@19.2.0) - vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + vite: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) optionalDependencies: '@tanstack/router-core': 1.136.8 transitivePeerDependencies: @@ -16961,14 +16967,14 @@ snapshots: - tsx - yaml - '@tanstack/router-devtools-core@1.136.8(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1)': + '@tanstack/router-devtools-core@1.136.8(@tanstack/router-core@1.136.8)(@types/node@24.10.1)(csstype@3.2.3)(jiti@1.21.7)(lightningcss@1.30.2)(solid-js@1.9.10)(tsx@4.20.6)(yaml@2.8.1)': dependencies: '@tanstack/router-core': 1.136.8 clsx: 2.1.1 goober: 2.1.18(csstype@3.2.3) solid-js: 1.9.10 tiny-invariant: 1.3.3 - vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + vite: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) optionalDependencies: csstype: 3.2.3 transitivePeerDependencies: @@ -17019,7 +17025,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@tanstack/router-plugin@1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1))': + '@tanstack/router-plugin@1.136.8(@tanstack/react-router@1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(vite@7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1))': dependencies: '@babel/core': 7.28.5 '@babel/plugin-syntax-jsx': 7.27.1(@babel/core@7.28.5) @@ -17037,7 +17043,7 @@ snapshots: zod: 3.25.76 optionalDependencies: '@tanstack/react-router': 1.136.8(react-dom@19.2.0(react@19.2.0))(react@19.2.0) - vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + vite: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) transitivePeerDependencies: - supports-color @@ -17923,7 +17929,7 @@ snapshots: '@vercel/oidc@3.0.3': {} - '@vitejs/plugin-react@4.7.0(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1))': + '@vitejs/plugin-react@4.7.0(vite@7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1))': dependencies: '@babel/core': 7.28.5 '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.28.5) @@ -17931,7 +17937,7 @@ snapshots: '@rolldown/pluginutils': 1.0.0-beta.27 '@types/babel__core': 7.20.5 react-refresh: 0.17.0 - vite: 7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + vite: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) transitivePeerDependencies: - supports-color @@ -17963,6 +17969,14 @@ snapshots: optionalDependencies: vite: 7.2.2(@types/node@22.19.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1))': + dependencies: + '@vitest/spy': 3.2.4 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + '@vitest/mocker@3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1))': dependencies: '@vitest/spy': 3.2.4 @@ -18397,6 +18411,10 @@ snapshots: astring@1.9.0: {} + async-mutex@0.5.0: + dependencies: + tslib: 2.8.1 + async-sema@3.1.1: {} async@3.2.6: {} @@ -25569,6 +25587,27 @@ snapshots: - tsx - yaml + vite-node@3.2.4(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1): + dependencies: + cac: 6.7.14 + debug: 4.4.3(supports-color@10.2.2) + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + vite-node@3.2.4(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1): dependencies: cac: 6.7.14 @@ -25617,6 +25656,22 @@ snapshots: tsx: 4.20.6 yaml: 2.8.1 + vite@7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1): + dependencies: + esbuild: 0.25.11 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.53.2 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 24.10.1 + fsevents: 2.3.3 + jiti: 1.21.7 + lightningcss: 1.30.2 + tsx: 4.20.6 + yaml: 2.8.1 + vite@7.2.2(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1): dependencies: esbuild: 0.25.11 @@ -25680,6 +25735,49 @@ snapshots: - tsx - yaml + vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.1)(jiti@1.21.7)(jsdom@27.2.0)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1): + dependencies: + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.4 + '@vitest/mocker': 3.2.4(vite@7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1)) + '@vitest/pretty-format': 3.2.4 + '@vitest/runner': 3.2.4 + '@vitest/snapshot': 3.2.4 + '@vitest/spy': 3.2.4 + '@vitest/utils': 3.2.4 + chai: 5.3.3 + debug: 4.4.3(supports-color@10.2.2) + expect-type: 1.2.2 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 7.2.2(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + vite-node: 3.2.4(@types/node@24.10.1)(jiti@1.21.7)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/debug': 4.1.12 + '@types/node': 24.10.1 + jsdom: 27.2.0 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + vitest@3.2.4(@types/debug@4.1.12)(@types/node@24.10.1)(jiti@2.6.1)(jsdom@27.2.0)(lightningcss@1.30.2)(tsx@4.20.6)(yaml@2.8.1): dependencies: '@types/chai': 5.2.3