From ebf44d0cf78543b9bc8b362de7c967d461623f4f Mon Sep 17 00:00:00 2001 From: Alexander Elsner Date: Fri, 10 Jul 2026 16:57:11 +0200 Subject: [PATCH 1/4] feat: implement attachment detection in IMAP body structure for efficient filtering --- src/utils/mails/backends/imap.ts | 153 ++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 35 deletions(-) diff --git a/src/utils/mails/backends/imap.ts b/src/utils/mails/backends/imap.ts index 3174ffc..a928d01 100644 --- a/src/utils/mails/backends/imap.ts +++ b/src/utils/mails/backends/imap.ts @@ -7,6 +7,29 @@ import { MailParser } from "../parser"; import { Logger } from "../../logger"; import { QuickSort } from "@cleverjs/utils"; +/** + * Walk an IMAP bodyStructure and report whether the message carries an + * attachment (a part explicitly dispositioned as an attachment, or a named + * non-inline part). Lets a `has:attachment` filter be evaluated from + * lightweight metadata, without downloading each message's full source. + */ +function bodyStructureHasAttachment(node: any): boolean { + if (!node || typeof node !== 'object') return false; + + const disposition = typeof node.disposition === 'string' + ? node.disposition.toLowerCase() + : undefined; + const filename = node.dispositionParameters?.filename ?? node.parameters?.name; + + if (disposition === 'attachment') return true; + if (filename && disposition !== 'inline') return true; + + if (Array.isArray(node.childNodes)) { + return node.childNodes.some((child: any) => bodyStructureHasAttachment(child)); + } + return false; +} + export class IMAPAccount { protected readonly client: ImapFlow; @@ -384,8 +407,6 @@ export class IMAPAccount { }); } - const allResults: IMAPAccount.CrossFolderSearchResult[] = []; - // Build IMAP search criteria const buildSearchCriteria = (): SearchObject => { const criteria: SearchObject = {}; @@ -456,12 +477,30 @@ export class IMAPAccount { const searchCriteria = buildSearchCriteria(); - // Search each mailbox + // `has:attachment` can't be expressed as IMAP SEARCH criteria, so it is + // evaluated from each match's bodyStructure during phase 1. + const needsAttachmentInfo = query.hasAttachment !== undefined; + + // ── Phase 1: collect lightweight metadata for every match ── + // Only envelope/date (+ bodyStructure when filtering by attachment) is + // fetched here — never the full message source — so we can sort and + // paginate across folders cheaply, even when a query matches thousands + // of messages. + interface LightMatch { + mailboxPath: string; + mailboxName: string; + specialUse?: string; + uid: number; + date: number; + } + + const matches: LightMatch[] = []; + for (const mailbox of mailboxesToSearch) { let lock; try { lock = await this.client.getMailboxLock(mailbox.path); - + const total = this.client.mailbox ? this.client.mailbox.exists : 0; if (total === 0) { lock.release(); @@ -469,7 +508,7 @@ export class IMAPAccount { } const searchResults = await this.client.search(searchCriteria, { uid: true }); - + // Ensure searchResults is an array (might be empty or non-array in edge cases) const uids: number[] = Array.isArray(searchResults) ? searchResults : []; @@ -478,30 +517,28 @@ export class IMAPAccount { continue; } - // Fetch mail details for matched UIDs - const rawMails = await this.client.fetchAll(uids.join(','), { + const metaMails = await this.client.fetchAll(uids.join(','), { + uid: true, envelope: true, - bodyStructure: true, - source: true, - flags: true + internalDate: true, + ...(needsAttachmentInfo ? { bodyStructure: true } : {}) }, { uid: true }); - let mails = await MailRessource.fromIMAPMessages(rawMails); + for (const meta of metaMails) { + if (needsAttachmentInfo) { + const hasAttachment = bodyStructureHasAttachment(meta.bodyStructure); + if (query.hasAttachment ? !hasAttachment : hasAttachment) { + continue; + } + } - // Post-fetch filtering for attachment - if (query.hasAttachment !== undefined) { - mails = mails.filter(mail => - query.hasAttachment ? mail.attachments.length > 0 : mail.attachments.length === 0 - ); - } - - // Add mailbox path to results - for (const mail of mails) { - allResults.push({ + const dateValue = meta.envelope?.date ?? meta.internalDate; + matches.push({ mailboxPath: mailbox.path, mailboxName: mailbox.name, specialUse: mailbox.specialUse, - mail + uid: meta.uid, + date: dateValue ? new Date(dateValue).getTime() : 0 }); } @@ -513,20 +550,66 @@ export class IMAPAccount { } } - // Sort all results by date - // allResults.sort((a, b) => { - // const dateA = a.mail.date || 0; - // const dateB = b.mail.date || 0; - // return order === 'newest' ? dateB - dateA : dateA - dateB; - // }); - QuickSort.sort(allResults, (a, b) => { - const dateA = a.mail.date ? a.mail.date : 0; - const dateB = b.mail.date ? b.mail.date : 0; - return order === 'newest' ? dateB - dateA : dateA - dateB; - }); + // Sort all matches by date, then take only the requested page. + QuickSort.sort(matches, (a, b) => order === 'newest' ? b.date - a.date : a.date - b.date); + + const pageMatches = matches.slice(offset, offset + limit); + if (pageMatches.length === 0) { + return []; + } + + // ── Phase 2: fetch full message source only for the page ── + // Grouped by folder so each mailbox is locked at most once. This is the + // only place the (expensive) source download + MIME parse happens, and + // it runs for at most `limit` messages instead of every match. + const pageByFolder = new Map(); + for (const match of pageMatches) { + const group = pageByFolder.get(match.mailboxPath); + if (group) group.push(match); + else pageByFolder.set(match.mailboxPath, [match]); + } + + const mailByKey = new Map(); + for (const [mailboxPath, group] of pageByFolder) { + let lock; + try { + lock = await this.client.getMailboxLock(mailboxPath); + + const rawMails = await this.client.fetchAll(group.map(m => m.uid).join(','), { + uid: true, + envelope: true, + bodyStructure: true, + source: true, + flags: true + }, { uid: true }); + + const mails = await MailRessource.fromIMAPMessages(rawMails); + for (const mail of mails) { + mailByKey.set(`${mailboxPath}:${mail.uid}`, mail); + } + + lock.release(); + } catch (e) { + if (lock) lock.release(); + Logger.error(`Failed to fetch mails in mailbox ${mailboxPath}`, e); + // Continue with other mailboxes + } + } + + // Rebuild results in the sorted page order. + const results: IMAPAccount.CrossFolderSearchResult[] = []; + for (const match of pageMatches) { + const mail = mailByKey.get(`${match.mailboxPath}:${match.uid}`); + if (!mail) continue; + results.push({ + mailboxPath: match.mailboxPath, + mailboxName: match.mailboxName, + specialUse: match.specialUse, + mail + }); + } - // Apply offset and limit across all results - return allResults.slice(offset, offset + limit); + return results; } } From 4fcdcd40e51b5848ff581b32e49b26a6f32aeea0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:44:35 +0000 Subject: [PATCH 2/4] fix: include inline CID parts in attachment bodyStructure detection --- src/utils/mails/backends/imap.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utils/mails/backends/imap.ts b/src/utils/mails/backends/imap.ts index a928d01..a50be66 100644 --- a/src/utils/mails/backends/imap.ts +++ b/src/utils/mails/backends/imap.ts @@ -20,9 +20,11 @@ function bodyStructureHasAttachment(node: any): boolean { ? node.disposition.toLowerCase() : undefined; const filename = node.dispositionParameters?.filename ?? node.parameters?.name; + const contentId = typeof node.id === 'string' ? node.id : undefined; if (disposition === 'attachment') return true; - if (filename && disposition !== 'inline') return true; + if (filename) return true; + if (disposition === 'inline' && contentId) return true; if (Array.isArray(node.childNodes)) { return node.childNodes.some((child: any) => bodyStructureHasAttachment(child)); From 461936a3df1a22142209fe3821d9bdf226813ad6 Mon Sep 17 00:00:00 2001 From: Linus Fischer Date: Fri, 10 Jul 2026 20:49:41 +0000 Subject: [PATCH 3/4] refactor: move bodyStructureHasAttachment function to static method in IMAPAccount class --- src/utils/mails/backends/imap.ts | 54 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/utils/mails/backends/imap.ts b/src/utils/mails/backends/imap.ts index a50be66..9cb4119 100644 --- a/src/utils/mails/backends/imap.ts +++ b/src/utils/mails/backends/imap.ts @@ -7,30 +7,6 @@ import { MailParser } from "../parser"; import { Logger } from "../../logger"; import { QuickSort } from "@cleverjs/utils"; -/** - * Walk an IMAP bodyStructure and report whether the message carries an - * attachment (a part explicitly dispositioned as an attachment, or a named - * non-inline part). Lets a `has:attachment` filter be evaluated from - * lightweight metadata, without downloading each message's full source. - */ -function bodyStructureHasAttachment(node: any): boolean { - if (!node || typeof node !== 'object') return false; - - const disposition = typeof node.disposition === 'string' - ? node.disposition.toLowerCase() - : undefined; - const filename = node.dispositionParameters?.filename ?? node.parameters?.name; - const contentId = typeof node.id === 'string' ? node.id : undefined; - - if (disposition === 'attachment') return true; - if (filename) return true; - if (disposition === 'inline' && contentId) return true; - - if (Array.isArray(node.childNodes)) { - return node.childNodes.some((child: any) => bodyStructureHasAttachment(child)); - } - return false; -} export class IMAPAccount { @@ -528,7 +504,7 @@ export class IMAPAccount { for (const meta of metaMails) { if (needsAttachmentInfo) { - const hasAttachment = bodyStructureHasAttachment(meta.bodyStructure); + const hasAttachment = IMAPAccount.bodyStructureHasAttachment(meta.bodyStructure); if (query.hasAttachment ? !hasAttachment : hasAttachment) { continue; } @@ -613,6 +589,34 @@ export class IMAPAccount { return results; } + + + /** + * Walk an IMAP bodyStructure and report whether the message carries an + * attachment (a part explicitly dispositioned as an attachment, or a named + * non-inline part). Lets a `has:attachment` filter be evaluated from + * lightweight metadata, without downloading each message's full source. + */ + private static bodyStructureHasAttachment(node: any): boolean { + if (!node || typeof node !== 'object') return false; + + const disposition = typeof node.disposition === 'string' + ? node.disposition.toLowerCase() + : undefined; + const filename = node.dispositionParameters?.filename ?? node.parameters?.name; + const contentId = typeof node.id === 'string' ? node.id : undefined; + + if (disposition === 'attachment') return true; + if (filename) return true; + if (disposition === 'inline' && contentId) return true; + + if (Array.isArray(node.childNodes)) { + return node.childNodes.some((child: any) => this.bodyStructureHasAttachment(child)); + } + return false; + } + + } export namespace IMAPAccount { From 3f6a8618172a84fb88bb66a5c757d8a93b323d0e Mon Sep 17 00:00:00 2001 From: Linus Fischer Date: Fri, 10 Jul 2026 20:53:35 +0000 Subject: [PATCH 4/4] refactor: move buildSearchCriteria function to a static method in IMAPAccount class --- src/utils/mails/backends/imap.ts | 138 +++++++++++++++---------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/src/utils/mails/backends/imap.ts b/src/utils/mails/backends/imap.ts index 9cb4119..87388a7 100644 --- a/src/utils/mails/backends/imap.ts +++ b/src/utils/mails/backends/imap.ts @@ -385,75 +385,7 @@ export class IMAPAccount { }); } - // Build IMAP search criteria - const buildSearchCriteria = (): SearchObject => { - const criteria: SearchObject = {}; - - if (query.text) { - // Full-text search across subject, from, to, and body - - criteria.or = [ - { subject: query.text }, - { from: query.text }, - { to: query.text }, - { body: query.text } - ]; - } - - if (query.subject) { - criteria.subject = query.subject; - } - - if (query.from) { - criteria.from = query.from; - } - - if (query.to) { - criteria.to = query.to; - } - - if (query.body) { - criteria.body = query.body; - } - - if (query.since) { - criteria.since = new Date(query.since); - } - - if (query.before) { - criteria.before = new Date(query.before); - } - - if (query.hasAttachment !== undefined) { - // IMAP doesn't have a direct "has attachment" flag, but we can filter later - // For now, we'll handle this post-fetch - } - - // Flag-based search - if (query.seen !== undefined) { - criteria.seen = query.seen; - } - - if (query.flagged !== undefined) { - criteria.flagged = query.flagged; - } - - if (query.answered !== undefined) { - criteria.answered = query.answered; - } - - if (query.draft !== undefined) { - criteria.draft = query.draft; - } - - if (criteria.or && criteria.or.length === 0) { - delete criteria.or; - } - - return criteria; - }; - - const searchCriteria = buildSearchCriteria(); + const searchCriteria = IMAPAccount.buildSearchCriteria(query); // `has:attachment` can't be expressed as IMAP SEARCH criteria, so it is // evaluated from each match's bodyStructure during phase 1. @@ -590,6 +522,74 @@ export class IMAPAccount { return results; } + // Build IMAP search criteria + private static buildSearchCriteria(query: IMAPAccount.SearchQuery): SearchObject { + const criteria: SearchObject = {}; + + if (query.text) { + // Full-text search across subject, from, to, and body + + criteria.or = [ + { subject: query.text }, + { from: query.text }, + { to: query.text }, + { body: query.text } + ]; + } + + if (query.subject) { + criteria.subject = query.subject; + } + + if (query.from) { + criteria.from = query.from; + } + + if (query.to) { + criteria.to = query.to; + } + + if (query.body) { + criteria.body = query.body; + } + + if (query.since) { + criteria.since = new Date(query.since); + } + + if (query.before) { + criteria.before = new Date(query.before); + } + + if (query.hasAttachment !== undefined) { + // IMAP doesn't have a direct "has attachment" flag, but we can filter later + // For now, we'll handle this post-fetch + } + + // Flag-based search + if (query.seen !== undefined) { + criteria.seen = query.seen; + } + + if (query.flagged !== undefined) { + criteria.flagged = query.flagged; + } + + if (query.answered !== undefined) { + criteria.answered = query.answered; + } + + if (query.draft !== undefined) { + criteria.draft = query.draft; + } + + if (criteria.or && criteria.or.length === 0) { + delete criteria.or; + } + + return criteria; + }; + /** * Walk an IMAP bodyStructure and report whether the message carries an