From 30bfc54a6c434281f55042de259015e30501d88c Mon Sep 17 00:00:00 2001 From: Qiyuanqiii <2297740147@qq.com> Date: Sun, 12 Jul 2026 01:01:36 +0800 Subject: [PATCH] fix(store): isolate backend keyword query semantics --- src/core/hooks/auto-recall.ts | 122 ++++++++++++-------------- src/core/record/l1-dedup.ts | 6 +- src/core/store/keyword-query.test.ts | 75 ++++++++++++++++ src/core/store/sqlite.ts | 18 ++++ src/core/store/tcvdb.ts | 24 +++-- src/core/store/types.ts | 6 ++ src/core/tools/conversation-search.ts | 19 ++-- src/core/tools/memory-search.ts | 19 ++-- 8 files changed, 185 insertions(+), 104 deletions(-) create mode 100644 src/core/store/keyword-query.test.ts diff --git a/src/core/hooks/auto-recall.ts b/src/core/hooks/auto-recall.ts index 23c9237c..d00108cd 100644 --- a/src/core/hooks/auto-recall.ts +++ b/src/core/hooks/auto-recall.ts @@ -3,7 +3,7 @@ * before the agent starts processing. * * - Searches L1 memories using configurable strategy (keyword / embedding / hybrid) - * - keyword: FTS5 BM25 (requires FTS5; returns empty if unavailable) + * - keyword: backend-native keyword ranking (requires a keyword index) * - embedding: VectorStore cosine similarity * - hybrid: keyword + embedding merged with RRF * - L3 persona injection @@ -18,7 +18,6 @@ import { readSceneIndex } from "../scene/scene-index.js"; import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js"; import type { MemoryRecord } from "../record/l1-reader.js"; import type { IMemoryStore, L1SearchResult, L1FtsResult } from "../store/types.js"; -import { buildFtsQuery } from "../store/sqlite.js"; import type { EmbeddingService, EmbeddingCallOptions } from "../store/embedding.js"; import { sanitizeText } from "../../utils/sanitize.js"; import type { Logger } from "../types.js"; @@ -392,7 +391,7 @@ async function searchMemories( } // ============================ -// Strategy: Keyword (FTS5 BM25, no in-memory fallback) +// Strategy: Keyword (backend-native index, no in-memory fallback) // ============================ async function searchByKeyword( @@ -403,43 +402,39 @@ async function searchByKeyword( logger?: Logger, vectorStore?: IMemoryStore, ): Promise { - // Prefer FTS5 if available + // Prefer the store's keyword index if available. if (vectorStore?.isFtsAvailable()) { - const ftsQuery = buildFtsQuery(userText); - if (ftsQuery) { - logger?.debug?.(`${TAG} [keyword-fts] Using FTS5 BM25 search: query="${ftsQuery}"`); - const ftsResults = await vectorStore.searchL1Fts(ftsQuery, maxResults * 2); - if (ftsResults.length > 0) { + const ftsResults = await vectorStore.searchL1Keyword(userText, maxResults * 2); + if (ftsResults.length > 0) { + logger?.debug?.( + `${TAG} [keyword] Raw results (${ftsResults.length}): ` + + ftsResults.map((r) => `id=${r.record_id} score=${r.score.toFixed(6)}`).join(", "), + ); + const filtered = ftsResults + .filter((r) => r.score >= threshold) + .slice(0, maxResults); + + if (filtered.length > 0) { + logger?.debug?.(`${TAG} [keyword] Found ${filtered.length} results (from ${ftsResults.length} raw, threshold=${threshold})`); + return filtered.map((r) => formatMemoryLine(ftsResultToFormatable(r))); + } + + // Keyword relevance scores are unreliable when the document set is very + // small (e.g. 1–3 records) because IDF approaches 0. In that case, + // trust the backend's match + rank ordering and return the top results. + if (ftsResults.length <= maxResults) { logger?.debug?.( - `${TAG} [keyword-fts] FTS5 raw results (${ftsResults.length}): ` + - ftsResults.map((r) => `id=${r.record_id} score=${r.score.toFixed(6)}`).join(", "), + `${TAG} [keyword] All ${ftsResults.length} results below threshold=${threshold} ` + + `but document set is small — returning all matched results`, ); - const filtered = ftsResults - .filter((r) => r.score >= threshold) - .slice(0, maxResults); - - if (filtered.length > 0) { - logger?.debug?.(`${TAG} [keyword-fts] FTS5 found ${filtered.length} results (from ${ftsResults.length} raw, threshold=${threshold})`); - return filtered.map((r) => formatMemoryLine(ftsResultToFormatable(r))); - } - - // BM25 absolute scores are unreliable when the document set is very - // small (e.g. 1–3 records) because IDF approaches 0. In that case, - // trust FTS5's MATCH + rank ordering and return the top results anyway. - if (ftsResults.length <= maxResults) { - logger?.debug?.( - `${TAG} [keyword-fts] All ${ftsResults.length} results below threshold=${threshold} ` + - `but document set is small — returning all matched results`, - ); - return ftsResults.slice(0, maxResults).map((r) => formatMemoryLine(ftsResultToFormatable(r))); - } - logger?.debug?.(`${TAG} [keyword-fts] FTS5 returned 0 results above threshold (from ${ftsResults.length} raw)`); + return ftsResults.slice(0, maxResults).map((r) => formatMemoryLine(ftsResultToFormatable(r))); } + logger?.debug?.(`${TAG} [keyword] Returned 0 results above threshold (from ${ftsResults.length} raw)`); } } - // FTS5 not available or returned no results — skip in-memory fallback to avoid O(N) full scan - logger?.debug?.(`${TAG} [keyword] FTS5 unavailable or no results, skipping keyword search`); + // Keyword index not available or returned no results — skip O(N) fallback. + logger?.debug?.(`${TAG} [keyword] Keyword search unavailable or empty, skipping`); return []; } @@ -499,13 +494,13 @@ async function searchByEmbedding( // ============================ /** - * Hybrid search: run keyword (FTS5) and embedding in parallel, merge with + * Hybrid search: run backend-native keyword and embedding search in parallel, merge with * Reciprocal Rank Fusion (RRF) to combine rank lists. * * RRF score for a record at rank r = 1 / (k + r), where k=60 is a constant. * If a record appears in both lists, its RRF scores are summed. * - * If FTS5 is unavailable, the keyword side returns empty and RRF uses + * If keyword search is unavailable, the keyword side returns empty and RRF uses * embedding results only. */ async function searchHybrid( @@ -522,41 +517,38 @@ async function searchHybrid( const candidateK = maxResults * 3; // retrieve more for merging const [keywordResult, embeddingResult] = await Promise.all([ - // Keyword search: FTS5 only (no in-memory fallback) + // Backend-native keyword search (no in-memory fallback) (async () => { const tStart = performance.now(); try { - // Try FTS5 first + // Let the backend translate raw text to its native keyword query. if (vectorStore.isFtsAvailable()) { - const ftsQuery = buildFtsQuery(userText); - if (ftsQuery) { - const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK); - if (ftsResults.length > 0) { - logger?.debug?.(`${TAG} [hybrid-keyword-fts] FTS5 found ${ftsResults.length} candidates`); - // Convert FtsSearchResult to ScoredRecord for RRF merge - const records = ftsResults.map((r): ScoredRecord => ({ - record: { - id: r.record_id, - content: r.content, - type: r.type as MemoryRecord["type"], - priority: r.priority, - scene_name: r.scene_name, - source_message_ids: [], - metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {}, - timestamps: [r.timestamp_str].filter(Boolean), - createdAt: "", - updatedAt: "", - sessionKey: r.session_key, - sessionId: r.session_id, - }, - score: r.score, - })); - return { records, ms: performance.now() - tStart }; - } + const ftsResults = await vectorStore.searchL1Keyword(userText, candidateK); + if (ftsResults.length > 0) { + logger?.debug?.(`${TAG} [hybrid-keyword] Found ${ftsResults.length} candidates`); + // Convert FtsSearchResult to ScoredRecord for RRF merge + const records = ftsResults.map((r): ScoredRecord => ({ + record: { + id: r.record_id, + content: r.content, + type: r.type as MemoryRecord["type"], + priority: r.priority, + scene_name: r.scene_name, + source_message_ids: [], + metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {}, + timestamps: [r.timestamp_str].filter(Boolean), + createdAt: "", + updatedAt: "", + sessionKey: r.session_key, + sessionId: r.session_id, + }, + score: r.score, + })); + return { records, ms: performance.now() - tStart }; } } - // FTS5 not available or returned no results — skip in-memory fallback - logger?.debug?.(`${TAG} [hybrid-keyword] FTS5 unavailable or no results, skipping keyword part`); + // Keyword search not available or returned no results — skip O(N) fallback. + logger?.debug?.(`${TAG} [hybrid-keyword] Keyword search unavailable or empty, skipping keyword part`); return { records: [] as ScoredRecord[], ms: performance.now() - tStart }; } catch (err) { logger?.warn?.(`${TAG} Hybrid: keyword part failed: ${err instanceof Error ? err.message : String(err)}`); @@ -853,7 +845,7 @@ function vectorResultToFormatable(r: L1SearchResult): FormatableMemory { } /** - * Build a FormatableMemory from an FtsSearchResult (FTS5 keyword search path). + * Build a FormatableMemory from an FtsSearchResult (keyword search path). * Handles empty/invalid metadata_json, empty timestamp_str gracefully. */ function ftsResultToFormatable(r: L1FtsResult): FormatableMemory { diff --git a/src/core/record/l1-dedup.ts b/src/core/record/l1-dedup.ts index 0fb6bf62..094a8bbe 100644 --- a/src/core/record/l1-dedup.ts +++ b/src/core/record/l1-dedup.ts @@ -17,7 +17,6 @@ import type { CandidateMatch } from "../prompts/l1-dedup.js"; import { CleanContextRunner } from "../../utils/clean-context-runner.js"; import { sanitizeJsonForParse } from "../../utils/sanitize.js"; import type { IMemoryStore } from "../store/types.js"; -import { buildFtsQuery } from "../store/sqlite.js"; import type { EmbeddingService } from "../store/embedding.js"; import type { LLMRunner, Logger } from "../types.js"; @@ -265,9 +264,8 @@ async function findCandidatesByFts( const matches: CandidateMatch[] = []; for (const mem of memories) { - const ftsQuery = buildFtsQuery(mem.content); - if (ftsQuery) { - const ftsResults = await vectorStore.searchL1Fts(ftsQuery, 10); + const ftsResults = await vectorStore.searchL1Keyword(mem.content, 10); + if (ftsResults.length > 0) { // Filter out records from the current batch const candidates: MemoryRecord[] = ftsResults .filter((r) => !newRecordIds.has(r.record_id)) diff --git a/src/core/store/keyword-query.test.ts b/src/core/store/keyword-query.test.ts new file mode 100644 index 00000000..334d7306 --- /dev/null +++ b/src/core/store/keyword-query.test.ts @@ -0,0 +1,75 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { VectorStore, _resetJiebaForTest, _setJiebaForTest } from "./sqlite.js"; +import { TcvdbMemoryStore } from "./tcvdb.js"; + +describe("backend keyword query boundary", () => { + beforeEach(() => { + _setJiebaForTest(null); + }); + + afterEach(() => { + _resetJiebaForTest(); + }); + + it("builds SQLite MATCH syntax inside the L1 store", () => { + const store = new VectorStore(":memory:", 0); + const search = vi.spyOn(store, "searchL1Fts").mockReturnValue([]); + + try { + store.searchL1Keyword("alpha beta", 7); + expect(search).toHaveBeenCalledWith('"alpha" OR "beta"', 7); + } finally { + store.close(); + } + }); + + it("builds SQLite MATCH syntax inside the L0 store", () => { + const store = new VectorStore(":memory:", 0); + const search = vi.spyOn(store, "searchL0Fts").mockReturnValue([]); + + try { + store.searchL0Keyword("alpha beta", 9); + expect(search).toHaveBeenCalledWith('"alpha" OR "beta"', 9); + } finally { + store.close(); + } + }); + + it("keeps raw L1 text for TCVDB embedding and BM25 search", async () => { + const store = createTcvdbStore(); + const search = vi.spyOn(store, "searchL1HybridAsync").mockResolvedValue([]); + + await store.searchL1Keyword(" alpha beta ", 7); + + expect(search).toHaveBeenCalledWith({ queryText: "alpha beta", topK: 7 }); + }); + + it("keeps raw L0 text for TCVDB embedding and BM25 search", async () => { + const store = createTcvdbStore(); + const search = vi.spyOn(store, "searchL0HybridAsync").mockResolvedValue([]); + + await store.searchL0Keyword(" alpha beta ", 9); + + expect(search).toHaveBeenCalledWith({ queryText: "alpha beta", topK: 9 }); + }); + + it("does not dispatch whitespace-only keyword queries", async () => { + const store = createTcvdbStore(); + const search = vi.spyOn(store, "searchL1HybridAsync").mockResolvedValue([]); + + await expect(store.searchL1Keyword(" ", 5)).resolves.toEqual([]); + expect(search).not.toHaveBeenCalled(); + }); +}); + +function createTcvdbStore(): TcvdbMemoryStore { + return new TcvdbMemoryStore({ + url: "http://127.0.0.1:1", + username: "test", + apiKey: "test", + database: "test", + embeddingModel: "test", + timeout: 1, + }); +} diff --git a/src/core/store/sqlite.ts b/src/core/store/sqlite.ts index 02816e50..5d92d5bc 100644 --- a/src/core/store/sqlite.ts +++ b/src/core/store/sqlite.ts @@ -2045,6 +2045,15 @@ export class VectorStore implements IMemoryStore { return this.ftsAvailable; } + /** + * Backend-neutral keyword search on L1 records. + * Converts raw user text to SQLite FTS5 MATCH syntax at the storage boundary. + */ + searchL1Keyword(queryText: string, limit = 20): FtsSearchResult[] { + const ftsQuery = buildFtsQuery(queryText); + return ftsQuery ? this.searchL1Fts(ftsQuery, limit) : []; + } + /** * FTS5 keyword search on L1 records. * Returns top-`limit` results sorted by BM25 relevance (highest first). @@ -2094,6 +2103,15 @@ export class VectorStore implements IMemoryStore { } } + /** + * Backend-neutral keyword search on L0 conversation messages. + * Converts raw user text to SQLite FTS5 MATCH syntax at the storage boundary. + */ + searchL0Keyword(queryText: string, limit = VectorStore.FTS_DEFAULT_LIMIT): L0FtsSearchResult[] { + const ftsQuery = buildFtsQuery(queryText); + return ftsQuery ? this.searchL0Fts(ftsQuery, limit) : []; + } + /** * FTS5 keyword search on L0 conversation messages. * Returns top-`limit` results sorted by BM25 relevance (highest first). diff --git a/src/core/store/tcvdb.ts b/src/core/store/tcvdb.ts index cbef1889..34158698 100644 --- a/src/core/store/tcvdb.ts +++ b/src/core/store/tcvdb.ts @@ -644,13 +644,15 @@ export class TcvdbMemoryStore implements IMemoryStore { return []; } + async searchL1Keyword(queryText: string, limit?: number): Promise { + const query = queryText.trim(); + if (!query) return []; + return this.searchL1HybridAsync({ queryText: query, topK: limit }); + } + async searchL1Fts(ftsQuery: string, limit?: number): Promise { - // TCVDB has no pure FTS — use hybrid search with sparse-only path - // The ftsQuery is raw text, use it as queryText for hybrid - if (!ftsQuery) return []; - const results = await this.searchL1HybridAsync({ queryText: ftsQuery, topK: limit }); - // L1SearchResult and L1FtsResult have identical shapes - return results; + // Legacy entry point retained for callers that still use the FTS-shaped API. + return this.searchL1Keyword(ftsQuery, limit); } async searchL1Hybrid(params: { @@ -975,10 +977,14 @@ export class TcvdbMemoryStore implements IMemoryStore { return []; } + async searchL0Keyword(queryText: string, limit?: number): Promise { + const query = queryText.trim(); + if (!query) return []; + return this.searchL0HybridAsync({ queryText: query, topK: limit }); + } + async searchL0Fts(ftsQuery: string, limit?: number): Promise { - if (!ftsQuery) return []; - // Use hybrid search; L0SearchResult and L0FtsResult have identical shapes - return this.searchL0HybridAsync({ queryText: ftsQuery, topK: limit }); + return this.searchL0Keyword(ftsQuery, limit); } /** diff --git a/src/core/store/types.ts b/src/core/store/types.ts index 7cf62ce8..7e484dc6 100644 --- a/src/core/store/types.ts +++ b/src/core/store/types.ts @@ -267,6 +267,9 @@ export interface IMemoryStore { // ── L1 Search ──────────────────────────────────────────── searchL1Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string): MaybePromise; + /** Search L1 with raw user text; the backend owns its query syntax. */ + searchL1Keyword(queryText: string, limit?: number): MaybePromise; + /** Low-level backend FTS search retained for compatibility. */ searchL1Fts(ftsQuery: string, limit?: number): MaybePromise; searchL1Hybrid?(params: { query?: string; @@ -293,6 +296,9 @@ export interface IMemoryStore { // ── L0 Search ──────────────────────────────────────────── searchL0Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string): MaybePromise; + /** Search L0 with raw user text; the backend owns its query syntax. */ + searchL0Keyword(queryText: string, limit?: number): MaybePromise; + /** Low-level backend FTS search retained for compatibility. */ searchL0Fts(ftsQuery: string, limit?: number): MaybePromise; pullProfiles?(): Promise; diff --git a/src/core/tools/conversation-search.ts b/src/core/tools/conversation-search.ts index d25ae178..444afc9c 100644 --- a/src/core/tools/conversation-search.ts +++ b/src/core/tools/conversation-search.ts @@ -2,16 +2,15 @@ * conversation_search tool: Agent-callable tool for searching L0 conversation records. * * Supports three search strategies with automatic degradation: - * 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel, + * 1. **hybrid** (default) — keyword + vector embedding in parallel, * merged via Reciprocal Rank Fusion (RRF). * 2. **embedding** — pure vector similarity (when FTS5 is unavailable). - * 3. **fts** — pure FTS5 keyword search (when embedding is unavailable). + * 3. **fts** — pure backend keyword search (when embedding is unavailable). * * The tool is registered via `api.registerTool()` in index.ts. */ import type { IMemoryStore, L0SearchResult } from "../store/types.js"; -import { buildFtsQuery } from "../store/sqlite.js"; import type { EmbeddingService } from "../store/embedding.js"; import type { Logger } from "../types.js"; @@ -136,18 +135,12 @@ export async function executeConversationSearch(params: { // ── Run available search strategies in parallel ── const [ftsItems, vecItems] = await Promise.all([ - // FTS5 keyword search on L0 + // Backend-native keyword search on L0 (async (): Promise => { if (!hasFts) return []; try { - const ftsQuery = buildFtsQuery(query); - if (!ftsQuery) { - logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`); - return []; - } - logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); - const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK); - logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); + const ftsResults = await vectorStore.searchL0Keyword(query, candidateK); + logger?.debug?.(`${TAG} [hybrid-keyword] Returned ${ftsResults.length} candidates`); return ftsResults.map((r) => ({ id: r.record_id, session_key: r.session_key, @@ -158,7 +151,7 @@ export async function executeConversationSearch(params: { })); } catch (err) { logger?.warn?.( - `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + `${TAG} [hybrid-keyword] Search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, ); return []; } diff --git a/src/core/tools/memory-search.ts b/src/core/tools/memory-search.ts index e1d5c05f..9b6b3ffa 100644 --- a/src/core/tools/memory-search.ts +++ b/src/core/tools/memory-search.ts @@ -2,16 +2,15 @@ * memory_search tool: Agent-callable tool for searching L1 memory records. * * Supports three search strategies with automatic degradation: - * 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel, + * 1. **hybrid** (default) — keyword + vector embedding in parallel, * merged via Reciprocal Rank Fusion (RRF). * 2. **embedding** — pure vector similarity (when FTS5 is unavailable). - * 3. **fts** — pure FTS5 keyword search (when embedding is unavailable). + * 3. **fts** — pure backend keyword search (when embedding is unavailable). * * The tool is registered via `api.registerTool()` in index.ts. */ import type { IMemoryStore, L1SearchResult } from "../store/types.js"; -import { buildFtsQuery } from "../store/sqlite.js"; import type { EmbeddingService } from "../store/embedding.js"; import type { Logger } from "../types.js"; @@ -137,18 +136,12 @@ export async function executeMemorySearch(params: { // ── Run available search strategies in parallel ── const [ftsItems, vecItems] = await Promise.all([ - // FTS5 keyword search + // Backend-native keyword search (async (): Promise => { if (!hasFts) return []; try { - const ftsQuery = buildFtsQuery(query); - if (!ftsQuery) { - logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`); - return []; - } - logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); - const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK); - logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); + const ftsResults = await vectorStore.searchL1Keyword(query, candidateK); + logger?.debug?.(`${TAG} [hybrid-keyword] Returned ${ftsResults.length} candidates`); return ftsResults.map((r) => ({ id: r.record_id, content: r.content, @@ -161,7 +154,7 @@ export async function executeMemorySearch(params: { })); } catch (err) { logger?.warn?.( - `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + `${TAG} [hybrid-keyword] Search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, ); return []; }