diff --git a/src/core/store/fts-query-budget.test.ts b/src/core/store/fts-query-budget.test.ts new file mode 100644 index 00000000..8155d16a --- /dev/null +++ b/src/core/store/fts-query-budget.test.ts @@ -0,0 +1,61 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { + _resetJiebaForTest, + _setJiebaForTest, + buildFtsQuery, +} from "./sqlite.js"; + +describe("buildFtsQuery query budget", () => { + afterEach(() => { + _resetJiebaForTest(); + }); + + it("deduplicates fallback tokens while preserving first-seen order", () => { + _setJiebaForTest(null); + + expect(buildFtsQuery("alpha beta alpha beta gamma")).toBe( + '"alpha" OR "beta" OR "gamma"', + ); + }); + + it("caps fallback output at 64 unique tokens", () => { + _setJiebaForTest(null); + const raw = Array.from({ length: 100 }, (_, index) => `term${index}`).join(" "); + + const query = buildFtsQuery(raw); + const terms = query?.split(" OR ") ?? []; + + expect(terms).toHaveLength(64); + expect(terms[0]).toBe('"term0"'); + expect(terms[63]).toBe('"term63"'); + expect(query).not.toContain('"term64"'); + }); + + it("applies the same cap to tokenizer output", () => { + _setJiebaForTest({ + cutForSearch: () => Array.from({ length: 100 }, (_, index) => `token${index}`), + }); + + const terms = buildFtsQuery("ignored")?.split(" OR ") ?? []; + + expect(terms).toHaveLength(64); + expect(terms[0]).toBe('"token0"'); + expect(terms[63]).toBe('"token63"'); + }); + + it("deduplicates tokenizer output before spending the token budget", () => { + _setJiebaForTest({ + cutForSearch: () => [ + ...Array.from({ length: 100 }, () => "repeated"), + ...Array.from({ length: 70 }, (_, index) => `unique${index}`), + ], + }); + + const terms = buildFtsQuery("ignored")?.split(" OR ") ?? []; + + expect(terms).toHaveLength(64); + expect(terms[0]).toBe('"repeated"'); + expect(terms[63]).toBe('"unique62"'); + }); +}); diff --git a/src/core/store/sqlite.ts b/src/core/store/sqlite.ts index 02816e50..00e292ca 100644 --- a/src/core/store/sqlite.ts +++ b/src/core/store/sqlite.ts @@ -143,6 +143,9 @@ function requireNodeSqlite(): typeof import("node:sqlite") { // Lazy-loaded singleton: initialised on first call to `buildFtsQuery`. // If @node-rs/jieba is unavailable, falls back to Unicode-regex splitting. +/** Bound generated MATCH term count for untrusted or oversized input. */ +const FTS_MAX_QUERY_TOKENS = 64; + interface JiebaInstance { cutForSearch(text: string, hmm: boolean): string[]; } @@ -184,6 +187,9 @@ const ZH_STOP_WORDS = new Set([ * Falls back to Unicode-regex splitting (`/[\p{L}\p{N}_]+/gu`) if * jieba is not installed. * + * Duplicate terms are removed in first-seen order and output is limited to + * 64 terms so oversized input cannot create an unbounded number of OR clauses. + * * Tokens are OR-joined as quoted FTS5 phrase terms so that a document * matching *any* token is returned. BM25 naturally ranks documents that * match more tokens higher, so precision is preserved while recall is @@ -225,7 +231,10 @@ export function buildFtsQuery(raw: string): string | null { } if (tokens.length === 0) return null; - const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`); + // Apply one final first-seen dedupe so fallback and tokenizer behavior match. + const quoted = [...new Set(tokens)] + .slice(0, FTS_MAX_QUERY_TOKENS) + .map((t) => `"${t.replaceAll('"', "")}"`); return quoted.join(" OR "); }