Merge branch 'main' into dev/robgruen/android

ts/examples/chat/package.json

@@ -31,6 +31,7 @@
     "dotenv": "^16.3.1",
     "interactive-app": "workspace:*",
     "knowledge-processor": "workspace:*",
+    "memory-providers": "workspace:*",
     "typeagent": "workspace:*",
     "typechat": "^0.1.1",
     "typescript": "^5.4.2"

ts/examples/chat/src/memory/chatMemory.ts

@@ -1271,10 +1271,9 @@ export async function runChatMemory(): Promise<void> {
     }
 
     function writeResultLinks(rr: conversation.SearchResponse): void {
-        if (rr && rr.messageIds) {
-            const links = rr.messageIds.map((id) =>
-                context.conversation.messages.getUrl(id).toString(),
-            );
+        const urlGet = context.conversation.messages.getUrl;
+        if (rr && rr.messageIds && urlGet !== undefined) {
+            const links = rr.messageIds.map((id) => urlGet(id).toString());
             printer.writeList(links, { type: "ul" });
         }
     }

ts/examples/memoryProviders/src/sqlite/common.ts

@@ -17,6 +17,8 @@ export type AssignedId<T> = {
     isNew: boolean;
 };
 
+export type BooleanRow = {};
+
 export async function createDb(
     filePath: string,
     createNew: boolean,
@@ -29,17 +31,33 @@ export async function createDb(
     return db;
 }
 
-export function createInQuery(
-    db: sqlite.Database,
-    tableName: string,
-    selectCol: string,
-    testCol: string,
-    values: any[],
-): sqlite.Statement {
-    const sql = `SELECT ${selectCol} from ${tableName} WHERE ${testCol} IN (${values})`;
-    return db.prepare(sql);
-}
-
 export function tablePath(rootName: string, name: string): string {
     return rootName + "_" + name;
 }
+
+export function sql_makeInClause(values: any[]): string {
+    let sql = "";
+    for (let i = 0; i < values.length; ++i) {
+        if (i > 0) {
+            sql += ", ";
+        }
+        sql += `'${values[i]}'`;
+    }
+    return sql;
+}
+
+export type ColumnSerializer = {
+    serialize: (x: any) => any;
+    deserialize: (x: any) => any;
+};
+
+export function getTypeSerializer<T extends ColumnType>(
+    type: SqlColumnType<T>,
+): [boolean, ColumnSerializer] {
+    const isIdInt = type === "INTEGER";
+    const serializer: ColumnSerializer = {
+        serialize: isIdInt ? (x: any) => x : (x: any) => x.toString(),
+        deserialize: isIdInt ? (x: any) => x : (x: any) => Number.parseInt(x),
+    };
+    return [isIdInt, serializer];
+}

ts/examples/memoryProviders/src/sqlite/index.ts

@@ -5,3 +5,4 @@ export * from "./common.js";
 export * from "./textTable.js";
 export * from "./keyValueTable.js";
 export * from "./vectorTable.js";
+export * from "./temporalTable.js";

ts/examples/memoryProviders/src/sqlite/keyValueTable.ts

@@ -10,14 +10,24 @@ export interface KeyValueTable<
     TKeyId extends ColumnType = string,
     TValueId extends ColumnType = string,
 > extends knowLib.KeyValueIndex<TKeyId, TValueId> {
+    readonly schemaSql: string;
+    readonly tableName: string;
+
     getSync(id: TKeyId): TValueId[] | undefined;
+    putSync(postings: TValueId[], id: TKeyId): TKeyId;
     iterate(id: TKeyId): IterableIterator<TValueId> | undefined;
     iterateScored(
         id: TKeyId,
         score?: number,
     ): IterableIterator<ScoredItem<TValueId>> | undefined;
     iterateMultiple(ids: TKeyId[]): IterableIterator<TValueId> | undefined;
-    putSync(postings: TValueId[], id: TKeyId): TKeyId;
+    iterateMultipleScored(
+        items: ScoredItem<TKeyId>[],
+    ): IterableIterator<ScoredItem<TValueId>>;
+    getHits(
+        ids: TKeyId[],
+        join?: string,
+    ): IterableIterator<ScoredItem<TValueId>>;
 }
 
 export function createKeyValueTable<
@@ -44,17 +54,25 @@ export function createKeyValueTable<
     const sql_get = db.prepare(
         `SELECT valueId from ${tableName} WHERE keyId = ? ORDER BY valueId ASC`,
     );
+    const sql_getScored = db.prepare(
+        `SELECT valueId as item, @score as score 
+        FROM ${tableName} WHERE keyId = @keyId ORDER BY valueId ASC`,
+    );
     const sql_add = db.prepare(
         `INSERT OR IGNORE INTO ${tableName} (keyId, valueId) VALUES (?, ?)`,
     );
     const sql_remove = db.prepare(`DELETE FROM ${tableName} WHERE keyId = ?`);
     return {
+        schemaSql,
+        tableName,
         get,
         getSync,
         getMultiple,
+        getHits,
         iterate,
         iterateScored,
         iterateMultiple,
+        iterateMultipleScored,
         put,
         putSync,
         replace,
@@ -73,8 +91,8 @@ export function createKeyValueTable<
         return rows.length > 0 ? rows.map((r) => r.valueId) : undefined;
     }
 
-    function* iterate(id: TKeyId): IterableIterator<TValueId> | undefined {
-        const rows = sql_get.iterate(id);
+    function* iterate(keyId: TKeyId): IterableIterator<TValueId> | undefined {
+        const rows = sql_get.iterate(keyId);
         let count = 0;
         for (const row of rows) {
             yield (row as KeyValueRow).valueId;
@@ -85,20 +103,14 @@ export function createKeyValueTable<
         }
     }
 
-    function* iterateScored(
-        id: TKeyId,
+    function iterateScored(
+        keyId: TKeyId,
         score: number,
-    ): IterableIterator<ScoredItem<TValueId>> | undefined {
-        const rows = sql_get.iterate(id);
-        let count = 0;
-        for (const row of rows) {
-            let item = (row as KeyValueRow).valueId;
-            yield { score, item };
-            ++count;
-        }
-        if (count === 0) {
-            return undefined;
-        }
+    ): IterableIterator<ScoredItem<TValueId>> {
+        return sql_getScored.iterate({
+            score: score,
+            keyId,
+        }) as IterableIterator<ScoredItem<TValueId>>;
     }
 
     function* iterateMultiple(
@@ -111,7 +123,9 @@ export function createKeyValueTable<
             return iterate(ids[0]);
         }
 
-        const sql = `SELECT DISTINCT valueId from ${tableName} WHERE keyId IN (${ids}) ORDER BY valueId ASC`;
+        const sql = `SELECT DISTINCT valueId FROM ${tableName} 
+        WHERE keyId IN (${ids}) 
+        ORDER BY valueId ASC`;
         const stmt = db.prepare(sql);
         const rows = stmt.iterate();
         let count = 0;
@@ -124,6 +138,14 @@ export function createKeyValueTable<
         }
     }
 
+    function iterateMultipleScored(
+        items: ScoredItem<TKeyId>[],
+    ): IterableIterator<ScoredItem<TValueId>> {
+        const sql = sql_multipleScored(items);
+        const stmt = db.prepare(sql);
+        return stmt.iterate() as IterableIterator<ScoredItem<TValueId>>;
+    }
+
     function getMultiple(
         ids: TKeyId[],
         concurrency?: number,
@@ -138,6 +160,25 @@ export function createKeyValueTable<
         return Promise.resolve(matches);
     }
 
+    function* getHits(
+        ids: TKeyId[],
+        join?: string,
+    ): IterableIterator<ScoredItem<TValueId>> {
+        const sql = join
+            ? `SELECT valueId AS item, count(*) AS score FROM ${tableName}
+        ${join} AND keyId IN (${ids})
+        GROUP BY valueId 
+        ORDER BY score DESC`
+            : `SELECT valueId AS item, count(*) AS score FROM ${tableName}
+        WHERE keyId IN (${ids})
+        GROUP BY valueId 
+        ORDER BY score DESC`;
+        const stmt = db.prepare(sql);
+        for (const row of stmt.iterate()) {
+            yield row as ScoredItem<TValueId>;
+        }
+    }
+
     function put(values: TValueId[], id?: TKeyId): Promise<TKeyId> {
         if (id === undefined) {
             // TODO: support
@@ -163,6 +204,27 @@ export function createKeyValueTable<
         return Promise.resolve();
     }
 
+    function sql_multipleScored(items: ScoredItem<TKeyId>[]) {
+        let sql = "SELECT item, SUM(score) AS score FROM (\n";
+        sql += sql_unionAllPostings(items);
+        sql += "\n)\n";
+        sql += "GROUP BY item";
+        return sql;
+    }
+
+    function sql_unionAllPostings(items: ScoredItem<TKeyId>[]) {
+        let sql = "";
+        for (const item of items) {
+            if (sql.length > 0) {
+                sql += "\nUNION ALL\n";
+            }
+            sql += `SELECT valueId as item, ${item.score} as score 
+            FROM ${tableName} WHERE keyId = ${item.item}`;
+        }
+        sql += "\nORDER BY valueId ASC";
+        return sql;
+    }
+
     type KeyValueRow = {
         keyId: TKeyId;
         valueId: TValueId;