image-attachment-support

apps/desktop/package.json

@@ -73,6 +73,7 @@
     "@xstate/react": "^6.0.0",
     "@xstate/store": "^3.11.2",
     "ai": "^5.0.93",
+    "async-mutex": "^0.5.0",
     "chroma-js": "^3.1.2",
     "clsx": "^2.1.1",
     "date-fns": "^4.1.0",

apps/desktop/src-tauri/capabilities/default.json

@@ -69,6 +69,12 @@
         { "path": "$DOWNLOAD/**/*" }
       ]
     },
+    {
+      "identifier": "fs:allow-remove",
+      "allow": [
+        { "path": "$DATA/hyprnote/**/*" }
+      ]
+    },
     "db2:default",
     "windows:default",
     "tracing:default",

apps/desktop/src/chat/resolve-attachments.ts

@@ -0,0 +1,52 @@
+import type { FileUIPart } from "ai";
+
+import { readChatAttachmentAsDataURL } from "../components/chat/attachments/storage";
+import type { HyprUIMessage } from "./types";
+
+export async function resolveChatFileReferences(
+  messages: HyprUIMessage[],
+  chatGroupId?: string,
+): Promise<HyprUIMessage[]> {
+  const resolved: HyprUIMessage[] = [];
+
+  for (const message of messages) {
+    const resolvedParts = await Promise.all(
+      message.parts.map(async (part) => {
+        if (part.type === "data-chat-file") {
+          if (!chatGroupId) {
+            return part;
+          }
+
+          const dataUrl = await readChatAttachmentAsDataURL(
+            chatGroupId,
+            part.data.attachmentId,
+          );
+
+          if (!dataUrl) {
+            return part;
+          }
+
+          return {
+            type: "file",
+            filename: part.data.filename,
+            mediaType: part.data.mediaType,
+            url: dataUrl,
+          } satisfies FileUIPart;
+        }
+
+        if (part.type === "file" && part.url.startsWith("blob:")) {
+          return part;
+        }
+
+        return part;
+      }),
+    );
+
+    resolved.push({
+      ...message,
+      parts: resolvedParts,
+    });
+  }
+
+  return resolved;
+}

apps/desktop/src/chat/transport.ts

@@ -7,19 +7,26 @@ import {
 } from "ai";
 
 import { type ToolRegistry } from "../contexts/tool";
+import { resolveChatFileReferences } from "./resolve-attachments";
 import type { HyprUIMessage } from "./types";
 
 export class CustomChatTransport implements ChatTransport<HyprUIMessage> {
   constructor(
     private registry: ToolRegistry,
     private model: LanguageModel,
+    private chatGroupId?: string,
   ) {}
 
   sendMessages: ChatTransport<HyprUIMessage>["sendMessages"] = async (
     options,
   ) => {
     const tools = this.registry.getTools("chat");
 
+    const resolvedMessages = await resolveChatFileReferences(
+      options.messages,
+      this.chatGroupId,
+    );
+
     const agent = new Agent({
       model: this.model,
       tools,
@@ -34,7 +41,7 @@ export class CustomChatTransport implements ChatTransport<HyprUIMessage> {
     });
 
     const result = agent.stream({
-      messages: convertToModelMessages(options.messages),
+      messages: convertToModelMessages(resolvedMessages),
     });
 
     return result.toUIMessageStream({

apps/desktop/src/chat/types.ts

@@ -6,4 +6,18 @@ export const messageMetadataSchema = z.object({
 });
 
 export type MessageMetadata = z.infer<typeof messageMetadataSchema>;
-export type HyprUIMessage = UIMessage<MessageMetadata>;
+
+export type ChatFileReferencePart = {
+  type: "chat-file";
+  attachmentId: string;
+  filename: string;
+  mediaType: string;
+  size: number;
+  fileUrl: string;
+};
+
+export type ChatDataParts = {
+  "chat-file": ChatFileReferencePart;
+};
+
+export type HyprUIMessage = UIMessage<MessageMetadata, ChatDataParts>;

apps/desktop/src/components/chat/attachments/storage.ts

@@ -0,0 +1,52 @@
+import { ATTACHMENT_SIZE_LIMIT } from "../../../shared/attachments/constants";
+import {
+  createAttachmentStorage,
+  ManifestCorruptionError,
+  type ManifestEntry,
+} from "../../../shared/attachments/storage";
+
+export type PersistedChatAttachment = ManifestEntry & {
+  filePath: string;
+  fileUrl: string;
+};
+
+export { ManifestCorruptionError };
+
+const chatStorage = createAttachmentStorage<ManifestEntry>({
+  getBasePath: (groupId: string) => `hyprnote/chat/${groupId}`,
+  entityName: "chat group",
+  maxSize: ATTACHMENT_SIZE_LIMIT,
+  includeTitle: false,
+});
+
+export async function loadChatAttachments(
+  groupId: string,
+): Promise<PersistedChatAttachment[]> {
+  return await chatStorage.load(groupId);
+}
+
+export async function saveChatAttachment(
+  groupId: string,
+  file: File,
+  attachmentId = crypto.randomUUID(),
+): Promise<PersistedChatAttachment> {
+  return await chatStorage.save(groupId, file, {}, attachmentId);
+}
+
+export async function removeChatAttachment(
+  groupId: string,
+  attachmentId: string,
+) {
+  return await chatStorage.remove(groupId, attachmentId);
+}
+
+export async function removeChatGroupAttachments(groupId: string) {
+  return await chatStorage.removeAll(groupId);
+}
+
+export async function readChatAttachmentAsDataURL(
+  groupId: string,
+  attachmentId: string,
+): Promise<string | null> {
+  return await chatStorage.readAsDataURL(groupId, attachmentId);
+}