diff --git a/src/core/store/fts-expiry-sync.test.ts b/src/core/store/fts-expiry-sync.test.ts new file mode 100644 index 00000000..b79bead5 --- /dev/null +++ b/src/core/store/fts-expiry-sync.test.ts @@ -0,0 +1,100 @@ +import { afterEach, describe, expect, it } from "vitest"; +import type { MemoryRecord } from "../record/l1-writer.js"; +import type { L0Record } from "./types.js"; +import { + _resetJiebaForTest, + _setJiebaForTest, + buildFtsQuery, + VectorStore, +} from "./sqlite.js"; + +const EXPIRED_AT = "2024-01-01T00:00:00.000Z"; +const FRESH_AT = "2026-01-01T00:00:00.000Z"; +const CUTOFF = "2025-01-01T00:00:00.000Z"; + +let store: VectorStore | undefined; + +afterEach(() => { + store?.close(); + store = undefined; + _resetJiebaForTest(); +}); + +function createStore(): VectorStore { + const instance = new VectorStore(":memory:", 0); + instance.init(); + _setJiebaForTest(null); + store = instance; + return instance; +} + +function l1Record(index: number): MemoryRecord { + const at = index === 0 ? EXPIRED_AT : FRESH_AT; + return { + id: `l1-${index}`, + content: index === 0 ? "expiredl1token" : `freshl1token${index}`, + type: "persona", + priority: 50, + scene_name: "test", + source_message_ids: [], + metadata: {}, + timestamps: [at], + createdAt: at, + updatedAt: at, + sessionKey: "session", + sessionId: "session-id", + }; +} + +function l0Record(index: number): L0Record { + const at = index === 0 ? EXPIRED_AT : FRESH_AT; + return { + id: `l0-${index}`, + sessionKey: "session", + sessionId: "session-id", + role: "user", + messageText: index === 0 ? "expiredl0token" : `freshl0token${index}`, + recordedAt: at, + timestamp: Date.parse(at), + }; +} + +function query(text: string): string { + const result = buildFtsQuery(text); + if (!result) throw new Error(`Expected an FTS query for ${text}`); + return result; +} + +describe("TTL cleanup FTS synchronization", () => { + it("removes expired L1 records from keyword search", () => { + const db = createStore(); + for (let index = 0; index < 5; index++) { + expect(db.upsertL1(l1Record(index), undefined)).toBe(true); + } + + expect(db.searchL1Fts(query("expiredl1token"))).toHaveLength(1); + expect(db.deleteL1Expired(CUTOFF)).toBe(1); + + expect(db.countL1()).toBe(4); + expect(db.searchL1Fts(query("expiredl1token"))).toEqual([]); + expect(db.searchL1Fts(query("freshl1token1")).map((row) => row.record_id)).toEqual([ + "l1-1", + ]); + }); + + it("removes expired L0 records from keyword search", () => { + const db = createStore(); + for (let index = 0; index < 5; index++) { + expect(db.upsertL0(l0Record(index), undefined)).toBe(true); + } + + expect(db.searchL0Fts(query("expiredl0token"))).toHaveLength(1); + expect(db.deleteL0Expired(CUTOFF)).toBe(1); + + expect(db.countL0()).toBe(4); + expect(db.searchL0Fts(query("expiredl0token"))).toEqual([]); + expect(db.searchL0Fts(query("freshl0token1")).map((row) => row.record_id)).toEqual([ + "l0-1", + ]); + }); +}); diff --git a/src/core/store/sqlite.ts b/src/core/store/sqlite.ts index 02816e50..d87ee01a 100644 --- a/src/core/store/sqlite.ts +++ b/src/core/store/sqlite.ts @@ -391,11 +391,13 @@ export class VectorStore implements IMemoryStore { // Prepared statements — FTS5 L1 (initialized in init()) private stmtL1FtsInsert!: StatementSync; private stmtL1FtsDelete!: StatementSync; + private stmtL1FtsDeleteExpired!: StatementSync; private stmtL1FtsSearch!: StatementSync; // Prepared statements — FTS5 L0 (initialized in init()) private stmtL0FtsInsert!: StatementSync; private stmtL0FtsDelete!: StatementSync; + private stmtL0FtsDeleteExpired!: StatementSync; private stmtL0FtsSearch!: StatementSync; /** @@ -793,6 +795,14 @@ export class VectorStore implements IMemoryStore { this.stmtL1FtsDelete = this.db.prepare("DELETE FROM l1_fts WHERE record_id = ?"); + this.stmtL1FtsDeleteExpired = this.db.prepare(` + DELETE FROM l1_fts + WHERE record_id IN ( + SELECT record_id FROM l1_records + WHERE updated_time != '' AND updated_time < ? + ) + `); + this.stmtL1FtsSearch = this.db.prepare(` SELECT record_id, content_original AS content, type, priority, scene_name, session_key, session_id, timestamp_str, timestamp_start, timestamp_end, @@ -812,6 +822,14 @@ export class VectorStore implements IMemoryStore { this.stmtL0FtsDelete = this.db.prepare("DELETE FROM l0_fts WHERE record_id = ?"); + this.stmtL0FtsDeleteExpired = this.db.prepare(` + DELETE FROM l0_fts + WHERE record_id IN ( + SELECT record_id FROM l0_conversations + WHERE recorded_at != '' AND recorded_at < ? + ) + `); + this.stmtL0FtsSearch = this.db.prepare(` SELECT record_id, message_text_original AS message_text, session_key, session_id, role, recorded_at, timestamp, bm25(l0_fts) AS rank @@ -1274,7 +1292,7 @@ export class VectorStore implements IMemoryStore { * **Fault-tolerant**: returns 0 on failure. * TTL cleanup by updated_time. * - * Deletes expired rows from l1_records and matching vectors from l1_vec + * Deletes expired rows from l1_records and matching vector/FTS entries * in a single transaction to guarantee consistency. */ deleteL1Expired(cutoffIso: string): number { @@ -1305,6 +1323,11 @@ export class VectorStore implements IMemoryStore { this.db.exec("BEGIN"); try { + // Run before deleting metadata because the FTS predicate reads the + // expiring record IDs from l1_records. + if (this.ftsAvailable) { + this.stmtL1FtsDeleteExpired.run(cutoffIso); + } if (this.vecTablesReady) { this.db.prepare( "DELETE FROM l1_vec WHERE updated_time != '' AND updated_time < ?", @@ -1678,7 +1701,7 @@ export class VectorStore implements IMemoryStore { /** * TTL cleanup by recorded_at (ISO string) for L0 records. * - * Deletes expired rows from l0_conversations and matching vectors from l0_vec + * Deletes expired rows from l0_conversations and matching vector/FTS entries * in a single transaction to guarantee consistency. */ deleteL0Expired(cutoffIso: string): number { @@ -1710,6 +1733,11 @@ export class VectorStore implements IMemoryStore { this.db.exec("BEGIN"); try { + // Run before deleting metadata because the FTS predicate reads the + // expiring record IDs from l0_conversations. + if (this.ftsAvailable) { + this.stmtL0FtsDeleteExpired.run(cutoffIso); + } if (this.vecTablesReady) { this.db.prepare( "DELETE FROM l0_vec WHERE recorded_at != '' AND recorded_at < ?",