feat(export): paginate the non-session collections the frame guard's hint calls out

plugin/skills/agentmemory-mcp-tools/REFERENCE.md

@@ -18,7 +18,7 @@ agentmemory exposes 54 MCP tools. 8 are in the lean core set (`--tools core` or
 | `memory_consolidate` | yes | `tier`: string | Run the 4-tier memory consolidation pipeline (working -> episodic -> semantic -> procedural). |
 | `memory_crystallize` |  | `actionIds`*: string, `project`: string, `sessionId`: string | Compress completed action chains into compact crystal digests using LLM summarization. Extracts narrative, key outcomes, files affected, and lessons. |
 | `memory_diagnose` | yes | `categories`: string | Run health checks across all subsystems (actions, leases, sentinels, sketches, signals, sessions, memories, mesh). Identifies stuck, orphaned, and inconsistent state. |
-| `memory_export` |  | none | Export all memory data as JSON. |
+| `memory_export` |  | `maxSessions`: number, `offset`: number, `collectionLimit`: number, `collectionOffset`: number, `collections`: string | Export memory data as JSON. Past the transport size limit the export is refused, so use the paging arguments on a large store. |
 | `memory_facet_query` |  | `matchAll`: string, `matchAny`: string, `targetType`: string | Query targets by facet tags with AND/OR logic. Find all actions tagged priority:urgent AND team:backend. |
 | `memory_facet_tag` |  | `targetId`*: string, `targetType`*: string, `dimension`*: string, `value`*: string | Attach a structured tag (dimension:value) to an action, memory, or observation for multi-dimensional categorization. |
 | `memory_file_history` |  | `files`*: string, `sessionId`: string | Get past observations about specific files. |

src/functions/export-import.ts

@@ -56,20 +56,101 @@ async function runChunked<T>(
   }
 }
 
+// Every collection mem::export can put in the payload — the vocabulary
+// `?collections=` is matched against. sessions, observations and profiles
+// are absent on purpose: they are windowed by maxSessions/offset, and
+// profiles are derived from the session page rather than listed.
+const EXPORT_COLLECTION_NAMES: ReadonlySet<string> = new Set([
+  "memories",
+  "summaries",
+  "graphNodes",
+  "graphEdges",
+  "semanticMemories",
+  "proceduralMemories",
+  "actions",
+  "actionEdges",
+  "sentinels",
+  "sketches",
+  "crystals",
+  "facets",
+  "lessons",
+  "insights",
+  "routines",
+  "signals",
+  "checkpoints",
+  "accessLogs",
+]);
+
+// Absent means every collection — the behaviour before this parameter
+// existed. Present means the caller chose, so an empty or all-unknown
+// list selects nothing: falling back to everything there would turn a
+// client-side typo into the full multi-megabyte dump this parameter
+// exists to avoid. Unknown names are dropped rather than refused so a
+// client can name a collection an older build does not have and still
+// get the ones it does.
+function parseCollections(raw: unknown): ReadonlySet<string> | undefined {
+  if (raw === undefined || raw === null) return undefined;
+  const names = Array.isArray(raw) ? raw : String(raw).split(",");
+  return new Set(
+    names
+      .map((name) => String(name).trim())
+      .filter((name) => EXPORT_COLLECTION_NAMES.has(name)),
+  );
+}
+
 export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
-  sdk.registerFunction("mem::export", 
-    async (data?: { maxSessions?: number; offset?: number }) => {
+  sdk.registerFunction("mem::export",
+    async (data?: {
+      maxSessions?: number;
+      offset?: number;
+      collectionLimit?: number;
+      collectionOffset?: number;
+      collections?: string[] | string;
+    }) => {
       const rawMax = Number(data?.maxSessions);
       const maxSessions = Number.isFinite(rawMax) && rawMax > 0 ? Math.min(Math.floor(rawMax), 1000) : undefined;
       const rawOffset = Number(data?.offset);
       const offset = Number.isFinite(rawOffset) && rawOffset >= 0 ? Math.floor(rawOffset) : 0;
+      const rawCollectionLimit = Number(data?.collectionLimit);
+      const collectionLimit =
+        Number.isFinite(rawCollectionLimit) && rawCollectionLimit > 0
+          ? Math.floor(rawCollectionLimit)
+          : undefined;
+      const rawCollectionOffset = Number(data?.collectionOffset);
+      const collectionOffset =
+        Number.isFinite(rawCollectionOffset) && rawCollectionOffset >= 0
+          ? Math.floor(rawCollectionOffset)
+          : 0;
+
+      const collections = parseCollections(data?.collections);
+      const isSelected = (name: string): boolean =>
+        collections === undefined || collections.has(name);
+
+      // Records every collection's full size before slicing, so the
+      // caller can tell how far it still has to page even though the
+      // response only carries one window. Deselected collections are
+      // counted too: totals are what clients read for corpus size, and
+      // an allowlist is about what travels, not about what is known.
+      const collectionTotals: Record<string, number> = {};
+      const sliceCollection = <T>(name: string, rows: T[]): T[] => {
+        collectionTotals[name] = rows.length;
+        if (!isSelected(name)) return [];
+        if (collectionLimit === undefined) return rows;
+        return rows.slice(collectionOffset, collectionOffset + collectionLimit);
+      };
 
       const allSessions = await kv.list<Session>(KV.sessions);
       const paginatedSessions = maxSessions !== undefined
         ? allSessions.slice(offset, offset + maxSessions)
         : allSessions;
-      const memories = await kv.list<Memory>(KV.memories);
-      const summaries = await kv.list<SessionSummary>(KV.summaries);
+      const memories = sliceCollection(
+        "memories",
+        await kv.list<Memory>(KV.memories),
+      );
+      const summaries = sliceCollection(
+        "summaries",
+        await kv.list<SessionSummary>(KV.summaries),
+      );
 
       const observations: Record<string, CompressedObservation[]> = {};
       const obsResults = await Promise.all(
@@ -115,22 +196,22 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
         checkpoints,
         accessLogs,
       ] = await Promise.all([
-        kv.list<GraphNode>(KV.graphNodes).catch(() => []),
-        kv.list<GraphEdge>(KV.graphEdges).catch(() => []),
-        kv.list<SemanticMemory>(KV.semantic).catch(() => []),
-        kv.list<ProceduralMemory>(KV.procedural).catch(() => []),
-        kv.list<Action>(KV.actions).catch(() => []),
-        kv.list<ActionEdge>(KV.actionEdges).catch(() => []),
-        kv.list<Sentinel>(KV.sentinels).catch(() => []),
-        kv.list<Sketch>(KV.sketches).catch(() => []),
-        kv.list<Crystal>(KV.crystals).catch(() => []),
-        kv.list<Facet>(KV.facets).catch(() => []),
-        kv.list<Lesson>(KV.lessons).catch(() => []),
-        kv.list<Insight>(KV.insights).catch(() => []),
-        kv.list<Routine>(KV.routines).catch(() => []),
-        kv.list<Signal>(KV.signals).catch(() => []),
-        kv.list<Checkpoint>(KV.checkpoints).catch(() => []),
-        kv.list<AccessLogExport>(KV.accessLog).catch(() => []),
+        kv.list<GraphNode>(KV.graphNodes).catch(() => []).then((r) => sliceCollection("graphNodes", r)),
+        kv.list<GraphEdge>(KV.graphEdges).catch(() => []).then((r) => sliceCollection("graphEdges", r)),
+        kv.list<SemanticMemory>(KV.semantic).catch(() => []).then((r) => sliceCollection("semanticMemories", r)),
+        kv.list<ProceduralMemory>(KV.procedural).catch(() => []).then((r) => sliceCollection("proceduralMemories", r)),
+        kv.list<Action>(KV.actions).catch(() => []).then((r) => sliceCollection("actions", r)),
+        kv.list<ActionEdge>(KV.actionEdges).catch(() => []).then((r) => sliceCollection("actionEdges", r)),
+        kv.list<Sentinel>(KV.sentinels).catch(() => []).then((r) => sliceCollection("sentinels", r)),
+        kv.list<Sketch>(KV.sketches).catch(() => []).then((r) => sliceCollection("sketches", r)),
+        kv.list<Crystal>(KV.crystals).catch(() => []).then((r) => sliceCollection("crystals", r)),
+        kv.list<Facet>(KV.facets).catch(() => []).then((r) => sliceCollection("facets", r)),
+        kv.list<Lesson>(KV.lessons).catch(() => []).then((r) => sliceCollection("lessons", r)),
+        kv.list<Insight>(KV.insights).catch(() => []).then((r) => sliceCollection("insights", r)),
+        kv.list<Routine>(KV.routines).catch(() => []).then((r) => sliceCollection("routines", r)),
+        kv.list<Signal>(KV.signals).catch(() => []).then((r) => sliceCollection("signals", r)),
+        kv.list<Checkpoint>(KV.checkpoints).catch(() => []).then((r) => sliceCollection("checkpoints", r)),
+        kv.list<AccessLogExport>(KV.accessLog).catch(() => []).then((r) => sliceCollection("accessLogs", r)),
       ]);
 
       const exportData: ExportData = {
@@ -170,6 +251,22 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
         };
       }
 
+      if (collectionLimit !== undefined) {
+        exportData.collectionPagination = {
+          offset: collectionOffset,
+          limit: collectionLimit,
+          totals: collectionTotals,
+          // Only the selected collections can move the flag: a client
+          // that asked for six of eighteen has to be able to stop on
+          // hasMore instead of hand-rolling an early stop against
+          // totals, and graphEdges still having rows is not its problem.
+          hasMore: Object.entries(collectionTotals).some(
+            ([name, total]) =>
+              isSelected(name) && collectionOffset + collectionLimit < total,
+          ),
+        };
+      }
+
       const totalObs = Object.values(observations).reduce(
         (sum, arr) => sum + arr.length,
         0,
@@ -180,13 +277,19 @@ export function registerExportImportFunction(sdk: ISdk, kv: StateKV): void {
         observations: totalObs,
         memories: memories.length,
         summaries: summaries.length,
+        // Logged post-filter so a caller whose names all got dropped can
+        // see an empty selection here rather than guess why the payload
+        // came back empty while totals looked healthy.
+        collections: collections && [...collections],
       });
 
-      // Only session collections page on ?maxSessions/?offset, so a large
-      // store can exceed the transport cap even at ?maxSessions=1.
+      // Sessions and their observations page on ?maxSessions/?offset; the
+      // rest page on ?collectionLimit/?collectionOffset. A store can still
+      // exceed the cap if one session's observations do, since those are
+      // atomic.
       const oversized = checkPayloadFrameSize(
         exportData,
-        "narrow the range with ?maxSessions / ?offset, or export fewer collections; the non-session collections (memories, graph, semantic, actions, lessons, ...) are not yet paginated",
+        "narrow the range with ?maxSessions / ?offset, page the rest with ?collectionLimit / ?collectionOffset, or ask for a subset with ?collections=",
       );
       if (oversized) {
         logger.warn("Export exceeds transport frame limit", {

src/mcp/server.ts

@@ -362,7 +362,31 @@ export function registerMcpEndpoints(
           }
 
           case "memory_export": {
-            const result = await sdk.trigger({ function_id: "mem::export", payload: {} });
+            // The tool is where an agent actually reaches export, so the
+            // paging arguments have to survive this layer or a large store
+            // stays unreachable from MCP. `collections` is forwarded raw,
+            // empty string included: mem::export reads an empty selection
+            // as "no collections", which only stays distinguishable from
+            // an absent argument if this layer does not drop it.
+            const exportPayload: {
+              maxSessions?: number;
+              offset?: number;
+              collectionLimit?: number;
+              collectionOffset?: number;
+              collections?: string;
+            } = {};
+            for (const key of ["maxSessions", "collectionLimit"] as const) {
+              const n = Number(args[key]);
+              if (Number.isInteger(n) && n > 0) exportPayload[key] = n;
+            }
+            for (const key of ["offset", "collectionOffset"] as const) {
+              const n = Number(args[key]);
+              if (Number.isInteger(n) && n >= 0) exportPayload[key] = n;
+            }
+            if (typeof args.collections === "string") {
+              exportPayload.collections = args.collections;
+            }
+            const result = await sdk.trigger({ function_id: "mem::export", payload: exportPayload });
             return {
               status_code: 200,
               body: {

src/mcp/tools-registry.ts

@@ -189,8 +189,32 @@ export const CORE_TOOLS: McpToolDef[] = [
   },
   {
     name: "memory_export",
-    description: "Export all memory data as JSON.",
-    inputSchema: { type: "object", properties: {} },
+    description:
+      "Export memory data as JSON. Past the transport size limit the export is refused, so use the paging arguments on a large store.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        maxSessions: {
+          type: "number",
+          description: "Sessions per page (with their observations)",
+        },
+        offset: { type: "number", description: "Session offset" },
+        collectionLimit: {
+          type: "number",
+          description:
+            "Rows per page for memories, graph, lessons and the other top-level collections",
+        },
+        collectionOffset: {
+          type: "number",
+          description: "Row offset for the top-level collections",
+        },
+        collections: {
+          type: "string",
+          description:
+            "Comma-separated allowlist of top-level collections to return, e.g. memories,summaries,lessons. Unknown names are ignored. Omit for all of them; totals still cover every collection either way.",
+        },
+      },
+    },
   },
   {
     name: "memory_relations",

src/triggers/api.ts

@@ -1299,9 +1299,18 @@ export function registerApiTriggers(
       // real corpus (40 sessions × 34K observations × 8K memories) hit the
       // iii engine invocation timeout and `agentmemory status` reported 0.
       // Pass through the query-string pagination so callers can chunk.
+      const payload: {
+        maxSessions?: number;
+        offset?: number;
+        collectionLimit?: number;
+        collectionOffset?: number;
+        collections?: string;
+      } = {};
       const rawMax = req.query_params?.["maxSessions"];
       const rawOffset = req.query_params?.["offset"];
-      const payload: { maxSessions?: number; offset?: number } = {};
+      const rawCollectionLimit = req.query_params?.["collectionLimit"];
+      const rawCollectionOffset = req.query_params?.["collectionOffset"];
+      const rawCollections = req.query_params?.["collections"];
       if (typeof rawMax === "string") {
         const n = Number(rawMax);
         if (Number.isInteger(n) && n > 0) payload.maxSessions = n;
@@ -1310,6 +1319,20 @@ export function registerApiTriggers(
         const n = Number(rawOffset);
         if (Number.isInteger(n) && n >= 0) payload.offset = n;
       }
+      if (typeof rawCollectionLimit === "string") {
+        const n = Number(rawCollectionLimit);
+        if (Number.isInteger(n) && n > 0) payload.collectionLimit = n;
+      }
+      if (typeof rawCollectionOffset === "string") {
+        const n = Number(rawCollectionOffset);
+        if (Number.isInteger(n) && n >= 0) payload.collectionOffset = n;
+      }
+      // Forwarded raw, empty value included: mem::export owns the name
+      // vocabulary, and only it can tell "?collections=" (an explicit
+      // empty selection) from an absent parameter (every collection).
+      if (typeof rawCollections === "string") {
+        payload.collections = rawCollections;
+      }
       const result = await sdk.trigger({
         function_id: "mem::export",
         payload,

src/types.ts

@@ -306,6 +306,25 @@ export interface ExportPagination {
   hasMore: boolean;
 }
 
+// Pagination for the top-level collections (memories, graph, lessons, …).
+// Separate from ExportPagination because that one slices sessions and the
+// observations hanging off them, which left every other collection
+// unbounded: a corpus could reach a size where even ?maxSessions=1 was
+// undeliverable, making the export permanently impossible at any
+// parameter.
+export interface ExportCollectionPagination {
+  offset: number;
+  limit: number;
+  // Every collection, always — including the ones a `collections`
+  // allowlist kept out of this response. Clients read totals for corpus
+  // counts, not only to size their own walk.
+  totals: Record<string, number>;
+  // True when any *selected* collection has rows past this window. With
+  // no allowlist that is every collection, as before.
+  hasMore: boolean;
+}
+
+
 export interface ExportData {
   version: "0.3.0" | "0.4.0" | "0.5.0" | "0.6.0" | "0.6.1" | "0.7.0" | "0.7.2" | "0.7.3" | "0.7.4" | "0.7.5" | "0.7.6" | "0.7.7" | "0.7.9" | "0.8.0" | "0.8.1" | "0.8.2" | "0.8.3" | "0.8.4" | "0.8.5" | "0.8.6" | "0.8.7" | "0.8.8" | "0.8.9" | "0.8.10" | "0.8.11" | "0.8.12" | "0.8.13" | "0.9.0" | "0.9.1" | "0.9.2" | "0.9.3" | "0.9.4" | "0.9.5" | "0.9.6" | "0.9.7" | "0.9.8" | "0.9.9" | "0.9.10" | "0.9.11" | "0.9.12" | "0.9.13" | "0.9.14" | "0.9.15" | "0.9.16" | "0.9.17" | "0.9.18" | "0.9.19" | "0.9.20" | "0.9.21" | "0.9.22" | "0.9.23" | "0.9.24" | "0.9.25" | "0.9.26" | "0.9.27" | "0.9.28" | "0.9.29";
   exportedAt: string;
@@ -331,6 +350,7 @@ export interface ExportData {
   insights?: Insight[];
   accessLogs?: AccessLogExport[];
   pagination?: ExportPagination;
+  collectionPagination?: ExportCollectionPagination;
 }
 
 export interface AccessLogExport {