From 07adcb3f98bde93fd9fa098c51dc660a4d5a59dc Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 12:06:57 -0400 Subject: [PATCH 01/12] feat: add durable activity history --- README.md | 6 + docs/README.md | 1 + docs/activity-history.md | 85 ++ docs/automations.md | 7 + docs/reference.md | 10 + src/activity-history.test.ts | 802 ++++++++++++++ src/activity-history.ts | 1892 ++++++++++++++++++++++++++++++++++ src/ai-cli.test.ts | 34 + src/ai.ts | 83 +- src/evolution-loop.ts | 16 + src/paths.test.ts | 28 + src/paths.ts | 26 + 12 files changed, 2989 insertions(+), 1 deletion(-) create mode 100644 docs/activity-history.md create mode 100644 src/activity-history.test.ts create mode 100644 src/activity-history.ts diff --git a/README.md b/README.md index 3609892..bb10f18 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ fclt ai loop activity --project fclt ai loop activity --project --json fclt ai loop activity --all --json fclt ai loop resolve --json +fclt ai loop history --project --since 2026-01-01T00:00:00Z --json ``` The activity snapshot explains what agents captured, what sources were checked, @@ -219,6 +220,11 @@ Aggregate actionable items may carry an opaque locator. Resolution revalidates the exact current scope and lifecycle revision and returns a plain-language plan without guessing a root or performing a mutation. +Use `loop history` for a bounded multi-run timeline. It returns append-only +event lineage, opaque cross-scope identities, cursor pagination, and explicit +snapshot-only, pruned, corrupt, or truncated coverage. The current activity +snapshot remains the fast default. + ### 3. Inspect existing AI state Start read-only: diff --git a/docs/README.md b/docs/README.md index 818b04f..578f4ae 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,6 +22,7 @@ Start with [Writeback and evolution](./writeback-evolution.md) for the product's - [Managed mode](./managed-mode.md): when to let `fclt` write tool files, and how adoption works. - [Security and trust](./security-trust.md): source trust, audit, secrets, and commit hygiene. - [Automations](./automations.md): recurring Codex loops for learning review, evolution review, and tool-call audit. +- [Activity history](./activity-history.md): bounded multi-run timelines, lineage identities, retention, and privacy rules. - [Command reference](./reference.md): command groups and common flags. - [Roadmap](./roadmap.md): current product gaps and planned work. diff --git a/docs/activity-history.md b/docs/activity-history.md new file mode 100644 index 0000000..c5a30fb --- /dev/null +++ b/docs/activity-history.md @@ -0,0 +1,85 @@ +# Activity history + +`fclt ai loop activity` remains the fast current-state snapshot. Use `fclt ai +loop history` when a consumer needs to explain how an item changed across +runs. + +```bash +fclt ai loop history --json +fclt ai loop history --project --since 2026-01-01T00:00:00Z --json +fclt ai loop history --item family:SF-example --limit 50 --json +fclt ai loop history --scope-id project:0123456789abcdef --event verification --json +``` + +The command defaults to all configured scopes. Use `--global` or `--project` +for one scope. `--since`, `--until`, `--scope-id`, `--item`, and repeatable +`--event` flags narrow the query. Results are newest first. `--limit` accepts +1–200 records; pass `page.nextCursor` back through `--cursor` for the next +page. + +## Contract + +The JSON response is the version 1 `activity-history` contract. It contains: + +- immutable event deltas, not copies of the current activity card +- run records for the events on the current page +- opaque scope, resource, family, and proposal identities +- optional lineage heads when `--item` is present +- per-scope availability, corruption, migration, and retention state +- pagination and separate truncation metadata + +Event types are `run`, `discovery`, `observation`, `correlation`, +`disposition`, `proposal`, `review`, `application`, `verification`, +`effectiveness`, `regression`, `supersession`, and `resolution`. The `action` +field carries the concrete transition, such as `repeated`, `task`, `defer`, +`rejected`, `applied`, `verified`, `improved`, `regressed`, `superseded`, or +`resolved`. + +Every event references one recorded run. The run record carries scope, trigger, +status, loop revision, config revision, reconciliation window, and coverage +counts. Resource events carry only the changed field and bounded safe context. +Consumers should join on `runId` and `resource.id`; they should not infer +lineage from titles or internal IDs. + +When correlation merges or another transition supersedes a prior resource, +`relatedResourceIds` contains the opaque successor identity. This keeps branch +lineage collision-safe without exposing machine or source identity. + +An item filter can match a returned resource ID or a safe scope-local item, +family, or proposal ID. The result sets `lineage.ambiguous` when the same local +ID exists in more than one scope. Use the opaque `scopeId` to disambiguate it. + +## Storage and retention + +History is machine-local runtime state. Each completed, degraded, or failed +non-preview loop run writes one immutable segment. Rewriting the same run is +idempotent and a checksum mismatch is refused. Preview runs do not write +history. + +The default retention policy keeps 365 days, at most 10,000 events, and at most +2,000 current lineage heads per scope. Retention removes whole oldest segments +and least-recently observed heads. The manifest records pruned event, segment, +and head counts plus the last pruned event boundary. A query that asks for data +before that boundary reports partial coverage; an item-lineage query also +reports partial coverage after any head pruning. + +The first version does not backfill from activity snapshots, technical loop +reports, writeback journals, or proposal files. Existing installations begin +with a truthful `snapshot-only` boundary. A query is complete only when its +explicit `--since` value is inside the retained version 1 window and every +selected scope is readable. + +## Privacy and integration + +The portable response excludes canonical roots, runtime paths, raw source +payloads, credentials, URL query strings, and local file links. It reports +corrupt, missing, omitted, pruned, and scan-limited history instead of silently +dropping it. + +Core history reads and writes do not call external trackers or vendors. Export +is not part of this contract. A future export must be a separate explicit, +redacted, reviewable operation. + +Human interfaces should consume this CLI/JSON contract directly. They should +not reconstruct history from runtime files or treat a current activity snapshot +as a timeline. diff --git a/docs/automations.md b/docs/automations.md index 18583ad..1b1f55c 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -42,6 +42,7 @@ fclt ai loop enable --project fclt ai loop status --project --json fclt ai loop run --project --json fclt ai loop activity --json +fclt ai loop history --project --since 2026-01-01T00:00:00Z --json fclt ai loop disable --project ``` @@ -88,6 +89,12 @@ and next actions without absolute machine paths. Use `fclt ai loop activity configured project, or add `--global`/`--project` to inspect one scope. Use the technical loop report only for diagnostics. +Multi-run history is separate from the latest activity snapshot. Each +non-preview run appends an immutable event segment, and `loop history` provides +bounded timeline and lineage reads with explicit pagination, retention, +migration, and degraded-coverage metadata. Downstream interfaces should use +that command instead of reconstructing history from runtime files. + ## Suggested Cadence - daily or per-project `learning-review` for durable signal diff --git a/docs/reference.md b/docs/reference.md index 8a5ca56..06b8994 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -123,6 +123,7 @@ fclt ai loop disable [--dry-run] [--json] fclt ai loop status [--json] fclt ai loop activity [--all|--global|--project] [--json] fclt ai loop resolve [--json] +fclt ai loop history [--all|--global|--project] [--since ] [--until ] [--item ] [--scope-id ] [--event ] [--limit <1-200>] [--cursor ] [--json] fclt ai loop run [--since ] [--until ] [--source ] [--dry-run] [--scheduled] [--json] ``` @@ -163,6 +164,15 @@ Older or non-actionable items without a locator remain handoff-only. See [Activity action locators](./activity-action-locators.md) for versioning and error semantics. +`loop history` is the bounded version 1 multi-run timeline and lineage +contract. It stores immutable per-run event segments in machine-local runtime +state and returns delta events rather than copying current activity items. +Queries are newest first and cursor-paginated. Scope, time, item, and event-type +filters are explicit. Missing pre-history, retention pruning, corruption, and +bounded omissions remain visible in `coverage` and `truncation`. See +[Activity history](./activity-history.md) for the schema, identity, retention, +and privacy contract. + `loop enable` is an explicit opt-in that installs an fclt-owned Codex automation. The loop persists the full current queue, emits a delta for notifications, retries bounded reconciliation failures, reports scheduler diff --git a/src/activity-history.test.ts b/src/activity-history.test.ts new file mode 100644 index 0000000..ee6eab7 --- /dev/null +++ b/src/activity-history.test.ts @@ -0,0 +1,802 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import type { ActivityFeed, ActivityItem } from "./activity"; +import { + type ActivityHistoryEventType, + appendActivityHistory, + queryActivityHistory, +} from "./activity-history"; +import type { EvolutionLoopReport } from "./evolution-loop"; +import { + facultAiActivityHistoryManifestPath, + facultAiActivityHistorySegmentDir, + facultAiEvolutionLoopConfigPath, +} from "./paths"; +import type { ReconciliationReview } from "./reconciliation-types"; + +const temporaryRoots: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryRoots.splice(0).map(async (root) => { + await rm(root, { recursive: true, force: true }); + }) + ); +}); + +async function tempHome(): Promise { + const home = await mkdtemp(join(tmpdir(), "fclt-activity-history-")); + temporaryRoots.push(home); + return home; +} + +function signalItem(args?: { + state?: ActivityItem["state"]; + disposition?: ActivityItem["decision"]["disposition"]; + evidenceCount?: number; + sourceLabels?: string[]; + title?: string; + linkedWork?: string[]; + rationale?: string; + familyId?: string; +}): ActivityItem { + const evidenceCount = args?.evidenceCount ?? 1; + const familyId = args?.familyId ?? "SF-stable"; + return { + id: `family:${familyId}`, + kind: "signal", + categories: ["signal"], + title: args?.title ?? "Repeated verification gap", + state: args?.state ?? "open", + change: "changed", + firstSeenAt: "2026-01-01T00:00:00.000Z", + lastChangedAt: "2026-01-01T00:00:00.000Z", + sourceLabels: args?.sourceLabels ?? ["Writeback"], + evidence: { + count: evidenceCount, + types: ["session"], + writebackIds: Array.from( + { length: evidenceCount }, + (_, index) => `WB-${String(index + 1).padStart(5, "0")}` + ), + }, + observations: [], + omittedObservations: 0, + context: { + scope: "global", + targets: [], + links: [], + }, + decision: { + disposition: args?.disposition, + rationale: args?.rationale, + }, + linkedWork: args?.linkedWork ?? [], + approvalRequired: false, + nextAction: "Review the signal", + technical: { + queueId: `family:${familyId}`, + familyId, + }, + }; +} + +function proposalItem(args: { + id: string; + status: NonNullable; + state?: ActivityItem["state"]; + verification?: NonNullable["state"]; +}): ActivityItem { + return { + id: `proposal:${args.id}`, + kind: "proposal", + categories: ["evolution"], + title: `Proposal ${args.id}`, + state: args.state ?? "approval_needed", + change: "changed", + firstSeenAt: "2026-01-01T00:00:00.000Z", + lastChangedAt: "2026-01-01T00:00:00.000Z", + sourceLabels: [], + evidence: { count: 1, types: ["writeback"], writebackIds: ["WB-00001"] }, + observations: [], + omittedObservations: 0, + context: { scope: "global", targets: [], links: [] }, + decision: { proposalStatus: args.status }, + linkedWork: [], + approvalRequired: false, + verification: args.verification + ? { state: args.verification, attempts: 1 } + : undefined, + nextAction: "Review the proposal", + technical: { + queueId: `proposal:${args.id}`, + proposalId: args.id, + }, + }; +} + +function review(args: { + rootDir: string; + scope: "global" | "project"; + index: number; + since?: string; + until: string; +}): ReconciliationReview { + return { + version: 1, + reviewId: `review-${args.index}`, + generatedAt: args.until, + window: { + id: `window-${args.index}`, + mode: "window", + since: args.since ?? "2026-01-01T00:00:00.000Z", + until: args.until, + scope: args.scope, + rootDir: args.rootDir, + configDigest: "config-revision", + }, + coverageComplete: true, + degraded: false, + coverage: [], + decisions: [], + evidence: [], + signals: [], + resolvedEvidenceKeys: [], + unresolvedSignals: [], + linkedWork: [], + dispositionCounts: { + propose: 0, + "apply-local": 0, + task: 0, + "resolve-watch": 0, + defer: 0, + }, + artifactPath: "review-artifact", + }; +} + +function report(args: { + rootDir: string; + scope: "global" | "project"; + index: number; + recordedAt: string; + items: ActivityItem[]; + status?: EvolutionLoopReport["status"]; +}): EvolutionLoopReport { + const status = args.status ?? "complete"; + const feed: ActivityFeed = { + version: 1, + mode: "latest", + snapshot: "embedded", + generatedAt: args.recordedAt, + scope: args.scope, + run: { id: `LR-${args.index}`, status }, + coverage: { + complete: status === "complete", + checked: 1, + degraded: 0, + sources: [], + }, + counts: { + total: args.items.length, + needsAttention: args.items.length, + new: 0, + changed: args.items.length, + resolved: 0, + unchangedSuppressed: 0, + }, + items: args.items, + }; + return { + version: 1, + runId: `LR-${args.index}`, + generatedAt: args.recordedAt, + scope: args.scope, + projectRoot: + args.scope === "project" ? args.rootDir.slice(0, -4) : undefined, + status, + trigger: "manual", + generationBefore: args.index - 1, + generationAfter: args.index, + reviewId: `review-${args.index}`, + coverage: [], + coverageComplete: status === "complete", + queue: [], + delta: { + new: [], + changed: [], + resolved: [], + notifiable: [], + unchangedSuppressed: 0, + }, + mutations: [], + attempts: [{ attempt: 1, ok: true }], + artifactPath: "loop-artifact", + auditPath: "loop-audit", + activity: feed, + }; +} + +async function appendRun(args: { + home: string; + rootDir: string; + scope?: "global" | "project"; + index: number; + recordedAt: string; + items: ActivityItem[]; + retention?: { maxAgeDays?: number; maxEvents?: number; maxHeads?: number }; +}): Promise>> { + const scope = args.scope ?? "global"; + const loopReport = report({ + rootDir: args.rootDir, + scope, + index: args.index, + recordedAt: args.recordedAt, + items: args.items, + }); + return await appendActivityHistory({ + homeDir: args.home, + rootDir: args.rootDir, + report: loopReport, + review: review({ + rootDir: args.rootDir, + scope, + index: args.index, + until: args.recordedAt, + }), + configRevision: 3, + retention: args.retention, + }); +} + +describe("activity history", () => { + test("records the complete lifecycle as deltas instead of repeated snapshots", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + await appendRun({ + home, + rootDir, + index: 1, + recordedAt: "2026-01-01T01:00:00.000Z", + items: [ + signalItem({ + state: "deferred", + disposition: "defer", + evidenceCount: 2, + sourceLabels: ["Writeback", "Repository"], + }), + proposalItem({ id: "EV-alpha", status: "proposed" }), + proposalItem({ id: "EV-beta", status: "proposed" }), + proposalItem({ id: "EV-gamma", status: "proposed" }), + ], + }); + await appendRun({ + home, + rootDir, + index: 2, + recordedAt: "2026-01-02T01:00:00.000Z", + items: [ + signalItem({ + state: "approval_needed", + disposition: "propose", + evidenceCount: 3, + }), + proposalItem({ id: "EV-alpha", status: "rejected" }), + proposalItem({ + id: "EV-beta", + status: "applied", + state: "verification_pending", + verification: "pending", + }), + proposalItem({ + id: "EV-gamma", + status: "superseded", + state: "resolved", + }), + ], + }); + await appendRun({ + home, + rootDir, + index: 3, + recordedAt: "2026-01-03T01:00:00.000Z", + items: [ + signalItem({ + state: "resolved", + disposition: "resolve-watch", + evidenceCount: 3, + }), + proposalItem({ + id: "EV-beta", + status: "applied", + state: "resolved", + verification: "improved", + }), + proposalItem({ + id: "EV-delta", + status: "applied", + state: "regressed", + verification: "regressed", + }), + ], + }); + + const result = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-01-01T01:00:00.000Z", + limit: 200, + }); + const actions = new Set(result.events.map((event) => event.action)); + for (const expected of [ + "discovered", + "repeated", + "correlated", + "defer", + "propose", + "proposal-proposed", + "rejected", + "applied", + "verification-pending", + "verified", + "improved", + "regressed", + "superseded", + "watch", + "resolved", + ]) { + expect(actions.has(expected as never)).toBe(true); + } + expect(result.coverage.complete).toBe(true); + expect(result.runs[0]).toMatchObject({ + configRevision: 3, + window: { since: "2026-01-01T00:00:00.000Z" }, + coverage: { complete: true }, + }); + }); + + test("is idempotent across reruns and overlapping windows while preserving renamed lineage", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + const first = await appendRun({ + home, + rootDir, + index: 1, + recordedAt: "2026-02-01T00:00:00.000Z", + items: [ + signalItem({ + evidenceCount: 2, + sourceLabels: ["Branch A", "Branch B"], + }), + ], + }); + const duplicate = await appendRun({ + home, + rootDir, + index: 1, + recordedAt: "2026-02-01T00:00:00.000Z", + items: [ + signalItem({ + evidenceCount: 2, + sourceLabels: ["Branch A", "Branch B"], + }), + ], + }); + await appendRun({ + home, + rootDir, + index: 2, + recordedAt: "2026-02-02T00:00:00.000Z", + items: [ + signalItem({ + evidenceCount: 2, + sourceLabels: ["Branch A", "Branch B"], + }), + ], + }); + await appendRun({ + home, + rootDir, + index: 3, + recordedAt: "2026-02-03T00:00:00.000Z", + items: [ + signalItem({ title: "Renamed verification gap", evidenceCount: 2 }), + ], + }); + expect(first.appended).toBe(true); + expect(duplicate.appended).toBe(false); + + const result = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-02-01T00:00:00.000Z", + item: "family:SF-stable", + limit: 200, + }); + expect( + result.events.filter((event) => event.action === "discovered") + ).toHaveLength(1); + expect( + result.events.filter((event) => event.action === "correlated") + ).toHaveLength(1); + expect( + result.events.some((event) => event.action === "metadata-updated") + ).toBe(true); + expect(result.lineage?.resources).toHaveLength(1); + }); + + test("keeps duplicate internal ids collision-safe across global and project scopes", async () => { + const home = await tempHome(); + const globalRoot = join(home, ".ai"); + const projectRoot = join(home, "work", "sample", ".ai"); + await appendRun({ + home, + rootDir: globalRoot, + index: 1, + recordedAt: "2026-03-01T00:00:00.000Z", + items: [signalItem()], + }); + await appendRun({ + home, + rootDir: projectRoot, + scope: "project", + index: 1, + recordedAt: "2026-03-01T00:00:00.000Z", + items: [signalItem()], + }); + + const result = await queryActivityHistory({ + homeDir: home, + rootDir: globalRoot, + scope: "all", + since: "2026-03-01T00:00:00.000Z", + item: "family:SF-stable", + limit: 200, + }); + expect(result.lineage?.ambiguous).toBe(true); + expect(result.lineage?.resources).toHaveLength(2); + expect( + new Set(result.lineage?.resources.map((entry) => entry.scopeId)).size + ).toBe(2); + expect( + new Set(result.lineage?.resources.map((entry) => entry.resource.id)).size + ).toBe(2); + }); + + test("links a superseded correlation branch to its opaque successor", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + await appendRun({ + home, + rootDir, + index: 1, + recordedAt: "2026-03-10T00:00:00.000Z", + items: [ + signalItem({ familyId: "SF-primary" }), + signalItem({ familyId: "SF-branch" }), + ], + }); + await appendRun({ + home, + rootDir, + index: 2, + recordedAt: "2026-03-11T00:00:00.000Z", + items: [ + signalItem({ familyId: "SF-primary", evidenceCount: 2 }), + signalItem({ + familyId: "SF-branch", + state: "resolved", + linkedWork: ["merged:family:SF-primary"], + }), + ], + }); + + const result = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-03-10T00:00:00.000Z", + limit: 200, + }); + const primary = result.events.find( + (event) => event.resource?.itemId === "family:SF-primary" + )?.resource; + const superseded = result.events.find( + (event) => + event.action === "superseded" && + event.resource?.itemId === "family:SF-branch" + ); + expect(primary).toBeDefined(); + expect(superseded?.relatedResourceIds).toEqual([primary!.id]); + }); + + test("paginates deterministic bounded queries with time, item, scope, and event filters", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + for (let index = 1; index <= 4; index += 1) { + await appendRun({ + home, + rootDir, + index, + recordedAt: `2026-04-0${index}T00:00:00.000Z`, + items: [signalItem({ evidenceCount: index })], + }); + } + const first = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-04-01T00:00:00.000Z", + until: "2026-04-04T00:00:00.000Z", + item: "family:SF-stable", + eventTypes: ["observation"], + limit: 2, + }); + expect(first.events).toHaveLength(2); + expect(first.page.nextCursor).toBeDefined(); + expect(first.truncation.truncated).toBe(false); + const second = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-04-01T00:00:00.000Z", + until: "2026-04-04T00:00:00.000Z", + item: "family:SF-stable", + eventTypes: ["observation"], + limit: 2, + cursor: first.page.nextCursor, + }); + expect(second.events.length).toBeGreaterThan(0); + expect( + second.events.some((event) => + first.events.some((prior) => prior.id === event.id) + ) + ).toBe(false); + const equivalentOffsets = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-04-01T01:00:00+01:00", + until: "2026-04-01T00:00:00.000Z", + eventTypes: ["run"], + }); + expect(equivalentOffsets.events).toHaveLength(1); + await expect( + queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + cursor: "not-a-cursor", + }) + ).rejects.toThrow("Invalid activity history cursor"); + }); + + test("prunes whole immutable segments and reports the retained window explicitly", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + for (let index = 1; index <= 5; index += 1) { + await appendRun({ + home, + rootDir, + index, + recordedAt: `2026-05-0${index}T00:00:00.000Z`, + items: [], + retention: { maxAgeDays: 365, maxEvents: 2 }, + }); + } + const partial = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-05-01T00:00:00.000Z", + }); + expect(partial.coverage.complete).toBe(false); + expect(partial.coverage.scopes[0]?.prunedEvents).toBe(3); + expect(partial.events).toHaveLength(2); + const retained = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-05-04T00:00:00.001Z", + }); + expect(retained.coverage.complete).toBe(true); + }); + + test("bounds lineage heads and reports item-lineage pruning separately", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + await appendRun({ + home, + rootDir, + index: 1, + recordedAt: "2026-05-10T00:00:00.000Z", + items: [ + signalItem({ familyId: "SF-primary" }), + signalItem({ familyId: "SF-branch" }), + ], + retention: { maxAgeDays: 365, maxEvents: 100, maxHeads: 1 }, + }); + + const timeline = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-05-10T00:00:00.000Z", + }); + expect(timeline.coverage.complete).toBe(true); + expect(timeline.coverage.scopes[0]?.prunedHeads).toBe(1); + const lineage = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-05-10T00:00:00.000Z", + item: "family:SF-primary", + }); + expect(lineage.coverage.complete).toBe(false); + }); + + test("reports snapshot-only migration and corrupt history without reconstructing other runtime files", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + const configPath = facultAiEvolutionLoopConfigPath(home, rootDir); + await mkdir(join(configPath, ".."), { recursive: true }); + await writeFile( + configPath, + `${JSON.stringify({ version: 1, scope: "global" })}\n`, + "utf8" + ); + const snapshotOnly = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + }); + expect(snapshotOnly.coverage.scopes[0]).toMatchObject({ + state: "snapshot-only", + detail: "history-not-recorded", + }); + expect(snapshotOnly.retention.migration).toBe( + "no-backfill-from-snapshots-or-journals" + ); + + const manifestPath = facultAiActivityHistoryManifestPath(home, rootDir); + await mkdir(dirname(manifestPath), { recursive: true }); + await writeFile(manifestPath, "{malformed", "utf8"); + const corrupt = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + }); + expect(corrupt.coverage.scopes[0]).toMatchObject({ + state: "degraded", + detail: "history-manifest-invalid", + }); + }); + + test("redacts private payloads and unsafe links and declares no export or external mutation", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + const credentialValue = ["private", "token"].join("-"); + const credentialText = ["Authorization", `Bearer ${credentialValue}`].join( + ": " + ); + const item = signalItem({ + title: `Failure at ${join(home, "work", "source.ts")}`, + rationale: `${credentialText} in ${rootDir}`, + }); + item.context = { + scope: "global", + targets: [], + links: [ + { + label: "unsafe", + url: `https://example.test/open?path=${encodeURIComponent(rootDir)}`, + source: "evidence", + }, + ], + }; + await appendRun({ + home, + rootDir, + index: 1, + recordedAt: "2026-06-01T00:00:00.000Z", + items: [item], + }); + const segmentDir = facultAiActivityHistorySegmentDir(home, rootDir); + const [segmentName] = await readdir(segmentDir); + const stored = await readFile(join(segmentDir, segmentName!), "utf8"); + expect(stored).not.toContain(home); + expect(stored).not.toContain(credentialValue); + expect(stored).not.toContain("?path="); + + const result = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + since: "2026-06-01T00:00:00.000Z", + }); + expect(result.capabilities).toEqual({ + externalMutation: false, + export: false, + rawPayloads: false, + }); + expect(JSON.stringify(result)).not.toContain(home); + }); + + test("isolates a corrupt segment and marks aggregate scope truncation", async () => { + const home = await tempHome(); + const globalRoot = join(home, ".ai"); + await appendRun({ + home, + rootDir: globalRoot, + index: 1, + recordedAt: "2026-07-01T00:00:00.000Z", + items: [], + }); + const segmentDir = facultAiActivityHistorySegmentDir(home, globalRoot); + const [segmentName] = await readdir(segmentDir); + const segmentPath = join(segmentDir, segmentName!); + const segment = JSON.parse(await readFile(segmentPath, "utf8")) as { + events: Array<{ action: string }>; + }; + segment.events[0]!.action = "unrecognized-action"; + const segmentBody = `${JSON.stringify(segment, null, 2)}\n`; + await writeFile(segmentPath, segmentBody, "utf8"); + const manifestPath = facultAiActivityHistoryManifestPath(home, globalRoot); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as { + segments: Array<{ checksum: string }>; + }; + manifest.segments[0]!.checksum = createHash("sha256") + .update(segmentBody) + .digest("hex"); + await writeFile( + manifestPath, + `${JSON.stringify(manifest, null, 2)}\n`, + "utf8" + ); + const corrupt = await queryActivityHistory({ + homeDir: home, + rootDir: globalRoot, + scope: "global", + since: "2026-07-01T00:00:00.000Z", + }); + expect(corrupt.coverage.scopes[0]).toMatchObject({ + state: "degraded", + corruptSegments: 1, + }); + + for (let index = 0; index < 51; index += 1) { + const projectRoot = join(home, "work", `project-${index}`, ".ai"); + await appendRun({ + home, + rootDir: projectRoot, + scope: "project", + index: 1, + recordedAt: "2026-07-02T00:00:00.000Z", + items: [], + }); + } + const aggregate = await queryActivityHistory({ + homeDir: home, + rootDir: globalRoot, + scope: "all", + since: "2026-07-02T00:00:00.000Z", + eventTypes: ["run"] as ActivityHistoryEventType[], + limit: 200, + }); + expect(aggregate.truncation.truncated).toBe(true); + expect(aggregate.truncation.omittedScopes).toBe(2); + expect(aggregate.coverage.complete).toBe(false); + }); +}); diff --git a/src/activity-history.ts b/src/activity-history.ts new file mode 100644 index 0000000..9f2e293 --- /dev/null +++ b/src/activity-history.ts @@ -0,0 +1,1892 @@ +import { createHash, randomUUID } from "node:crypto"; +import type { Dirent } from "node:fs"; +import { + lstat, + mkdir, + readdir, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { basename, dirname, join } from "node:path"; +import type { ActivityItem } from "./activity"; +import { redactPortableActivityText } from "./activity"; +import type { EvolutionLoopReport } from "./evolution-loop"; +import { + facultAiActivityHistoryManifestPath, + facultAiActivityHistorySegmentDir, + facultAiEvolutionLoopConfigPath, + facultLocalStateRoot, + machineStateProjectKey, + machineStateProjectScopeId, +} from "./paths"; +import type { ReconciliationReview } from "./reconciliation-types"; + +const HISTORY_VERSION = 1; +const DEFAULT_RETENTION_DAYS = 365; +const DEFAULT_RETENTION_EVENTS = 10_000; +const DEFAULT_RETENTION_HEADS = 2000; +const DEFAULT_QUERY_LIMIT = 50; +const MAX_QUERY_LIMIT = 200; +const MAX_MANIFEST_BYTES = 8_000_000; +const MAX_SEGMENT_BYTES = 4_000_000; +const MAX_DISCOVERED_PROJECTS = 1000; +const MAX_QUERY_SCOPES = 50; +const MAX_QUERY_SCANNED_EVENTS = 50_000; +const MAX_PORTABLE_TEXT = 1000; +const PORTABLE_IDENTIFIER_RE = /^[A-Za-z0-9][A-Za-z0-9:._-]{0,199}$/; +const SHA256_RE = /^[a-f0-9]{64}$/; + +export type ActivityHistoryEventType = + | "run" + | "discovery" + | "observation" + | "correlation" + | "disposition" + | "proposal" + | "review" + | "application" + | "verification" + | "effectiveness" + | "regression" + | "supersession" + | "resolution"; + +export type ActivityHistoryAction = + | "run-recorded" + | "discovered" + | "repeated" + | "metadata-updated" + | "linked-work-updated" + | "state-changed" + | "correlated" + | "correlation-split" + | "propose" + | "apply-local" + | "task" + | "watch" + | "defer" + | "proposal-proposed" + | "drafted" + | "in-review" + | "accepted" + | "rejected" + | "applied" + | "application-failed" + | "verification-unscheduled" + | "verification-pending" + | "verification-due" + | "verification-overdue" + | "verified" + | "improved" + | "unchanged" + | "regressed" + | "inconclusive" + | "superseded" + | "resolved"; + +const HISTORY_EVENT_TYPES = new Set([ + "run", + "discovery", + "observation", + "correlation", + "disposition", + "proposal", + "review", + "application", + "verification", + "effectiveness", + "regression", + "supersession", + "resolution", +]); + +const HISTORY_ACTIONS = new Set([ + "run-recorded", + "discovered", + "repeated", + "metadata-updated", + "linked-work-updated", + "state-changed", + "correlated", + "correlation-split", + "propose", + "apply-local", + "task", + "watch", + "defer", + "proposal-proposed", + "drafted", + "in-review", + "accepted", + "rejected", + "applied", + "application-failed", + "verification-unscheduled", + "verification-pending", + "verification-due", + "verification-overdue", + "verified", + "improved", + "unchanged", + "regressed", + "inconclusive", + "superseded", + "resolved", +]); + +const HISTORY_TRANSITION_FIELDS = new Set< + NonNullable["field"] +>([ + "state", + "evidence", + "correlation", + "disposition", + "proposal-status", + "verification", + "metadata", + "linked-work", +]); + +export interface ActivityHistoryResourceIdentity { + id: string; + kind: ActivityItem["kind"]; + itemId?: string; + familyId?: string; + proposalId?: string; +} + +export interface ActivityHistoryRun { + id: string; + recordedAt: string; + scopeId: string; + scope: "global" | "project"; + trigger: EvolutionLoopReport["trigger"]; + status: EvolutionLoopReport["status"]; + revision: number; + configRevision: number; + reviewId?: string; + window?: { + since: string; + until: string; + }; + coverage: { + complete: boolean; + checked: number; + degraded: number; + }; +} + +export interface ActivityHistoryEvent { + version: 1; + id: string; + recordedAt: string; + type: ActivityHistoryEventType; + action: ActivityHistoryAction; + scopeId: string; + runId: string; + resource?: ActivityHistoryResourceIdentity; + relatedResourceIds?: string[]; + transition?: { + field: + | "state" + | "evidence" + | "correlation" + | "disposition" + | "proposal-status" + | "verification" + | "metadata" + | "linked-work"; + from?: string; + to: string; + }; + context?: { + title: string; + evidenceCount: number; + sourceLabels: string[]; + rationale?: string; + target?: string; + linkedWork: string[]; + links: Array<{ label: string; url: string }>; + }; +} + +interface ActivityHistoryHead { + resource: ActivityHistoryResourceIdentity; + state: string; + disposition?: string; + proposalStatus?: string; + verification?: string; + evidenceFingerprint: string; + evidenceCount: number; + correlated: boolean; + titleFingerprint: string; + linkedWorkFingerprint: string; + firstRecordedAt: string; + lastRecordedAt: string; + eventCount: number; +} + +interface ActivityHistorySegmentRecord { + id: string; + file: string; + recordedAt: string; + eventCount: number; + checksum: string; +} + +interface ActivityHistoryManifest { + version: 1; + scopeId: string; + scope: "global" | "project"; + createdAt: string; + updatedAt: string; + migration: { + strategy: "no-backfill"; + snapshotOnlyBefore: string; + }; + retention: { + maxAgeDays: number; + maxEvents: number; + maxHeads: number; + prunedEvents: number; + prunedSegments: number; + prunedHeads: number; + prunedBefore?: string; + }; + segments: ActivityHistorySegmentRecord[]; + heads: Record; +} + +interface ActivityHistorySegment { + version: 1; + run: ActivityHistoryRun; + events: ActivityHistoryEvent[]; +} + +export interface ActivityHistoryQuery { + scope: "all" | "global" | "project"; + homeDir: string; + rootDir: string; + since?: string; + until?: string; + item?: string; + scopeId?: string; + eventTypes?: ActivityHistoryEventType[]; + limit?: number; + cursor?: string; +} + +export interface ActivityHistoryQueryResult { + version: 1; + kind: "activity-history"; + mode: "timeline"; + filters: { + scope: ActivityHistoryQuery["scope"]; + since?: string; + until?: string; + item?: string; + scopeId?: string; + eventTypes: ActivityHistoryEventType[]; + }; + capabilities: { + externalMutation: false; + export: false; + rawPayloads: false; + }; + coverage: { + state: "complete" | "partial" | "unavailable"; + complete: boolean; + configuredScopes: number; + reportingScopes: number; + degradedScopes: number; + snapshotOnlyScopes: number; + scopes: Array<{ + id: string; + scope: "global" | "project"; + state: + | "available" + | "snapshot-only" + | "degraded" + | "unavailable" + | "omitted"; + historyStart?: string; + snapshotOnlyBefore?: string; + prunedBefore?: string; + prunedEvents: number; + prunedHeads: number; + corruptSegments: number; + detail?: string; + }>; + }; + retention: { + defaultMaxAgeDays: number; + defaultMaxEventsPerScope: number; + defaultMaxLineageHeadsPerScope: number; + migration: "no-backfill-from-snapshots-or-journals"; + }; + events: ActivityHistoryEvent[]; + runs: ActivityHistoryRun[]; + lineage?: { + query: string; + ambiguous: boolean; + resources: Array<{ + scopeId: string; + resource: ActivityHistoryResourceIdentity; + firstRecordedAt: string; + lastRecordedAt: string; + eventCount: number; + current: { + state: string; + disposition?: string; + proposalStatus?: string; + verification?: string; + }; + }>; + }; + page: { + limit: number; + nextCursor?: string; + }; + truncation: { + truncated: boolean; + omittedScopes: number; + scanLimitReached: boolean; + }; +} + +interface ScopeDescriptor { + id: string; + scope: "global" | "project"; + configured: boolean; + manifestPath: string; + segmentDir: string; + omitted: boolean; +} + +interface ScopeRead { + descriptor: ScopeDescriptor; + manifest?: ActivityHistoryManifest; + events: ActivityHistoryEvent[]; + runs: ActivityHistoryRun[]; + state: ActivityHistoryQueryResult["coverage"]["scopes"][number]["state"]; + detail?: string; + corruptSegments: number; + scanLimitReached: boolean; +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isIsoDate(value: unknown): value is string { + return typeof value === "string" && Number.isFinite(Date.parse(value)); +} + +function dateValue(value: string): number { + return Date.parse(value); +} + +function isNonNegativeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && Number(value) >= 0; +} + +function isPortableIdentifier(value: unknown): value is string { + return typeof value === "string" && PORTABLE_IDENTIFIER_RE.test(value); +} + +function boundedText(value: string): string { + return redactPortableActivityText(value).slice(0, MAX_PORTABLE_TEXT); +} + +function portableIdentifier(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + const redacted = redactPortableActivityText(value); + return redacted === value && PORTABLE_IDENTIFIER_RE.test(value) + ? value + : undefined; +} + +function opaqueIdentity( + kind: "resource" | "family" | "proposal", + scopeId: string, + value: string +): string { + return `${kind}:${sha256(`${scopeId}\n${value}`).slice(0, 24)}`; +} + +function safeLink(value: { label: string; url: string }): { + label: string; + url: string; +} | null { + try { + const parsed = new URL(value.url); + if ( + (parsed.protocol !== "https:" && parsed.protocol !== "http:") || + parsed.username || + parsed.password + ) { + return null; + } + parsed.search = ""; + parsed.hash = ""; + const url = parsed.toString(); + if (boundedText(url) !== url) { + return null; + } + return { label: boundedText(value.label), url }; + } catch { + return null; + } +} + +function resourceIdentity( + scopeId: string, + item: ActivityItem +): ActivityHistoryResourceIdentity { + const stableSeed = + item.technical.familyId ?? + item.technical.proposalId ?? + item.technical.queueId; + return { + id: opaqueIdentity("resource", scopeId, `${item.kind}:${stableSeed}`), + kind: item.kind, + itemId: portableIdentifier(item.id), + familyId: item.technical.familyId + ? opaqueIdentity("family", scopeId, item.technical.familyId) + : undefined, + proposalId: item.technical.proposalId + ? opaqueIdentity("proposal", scopeId, item.technical.proposalId) + : undefined, + }; +} + +function eventContext(item: ActivityItem): ActivityHistoryEvent["context"] { + return { + title: boundedText(item.title), + evidenceCount: item.evidence.count, + sourceLabels: item.sourceLabels.map(boundedText).slice(0, 10), + rationale: item.decision.rationale + ? boundedText(item.decision.rationale) + : undefined, + target: item.decision.target + ? boundedText(item.decision.target) + : undefined, + linkedWork: item.linkedWork.map(boundedText).slice(0, 10), + links: (item.context?.links ?? []) + .map(safeLink) + .filter((entry): entry is { label: string; url: string } => + Boolean(entry) + ) + .slice(0, 5), + }; +} + +function historyEvent(args: { + run: ActivityHistoryRun; + type: ActivityHistoryEventType; + action: ActivityHistoryAction; + resource?: ActivityHistoryResourceIdentity; + relatedResourceIds?: string[]; + transition?: ActivityHistoryEvent["transition"]; + context?: ActivityHistoryEvent["context"]; +}): ActivityHistoryEvent { + const seed = JSON.stringify({ + runId: args.run.id, + type: args.type, + action: args.action, + resourceId: args.resource?.id, + relatedResourceIds: args.relatedResourceIds, + transition: args.transition, + }); + return { + version: 1, + id: `event:${sha256(`${args.run.scopeId}\n${seed}`).slice(0, 24)}`, + recordedAt: args.run.recordedAt, + type: args.type, + action: args.action, + scopeId: args.run.scopeId, + runId: args.run.id, + resource: args.resource, + relatedResourceIds: args.relatedResourceIds, + transition: args.transition, + context: args.context, + }; +} + +function dispositionAction(value: string): ActivityHistoryAction { + if (value === "resolve-watch") { + return "watch"; + } + if ( + value === "propose" || + value === "apply-local" || + value === "task" || + value === "defer" + ) { + return value; + } + return "state-changed"; +} + +function proposalTransition(value: string): { + type: ActivityHistoryEventType; + action: ActivityHistoryAction; +} { + if (value === "proposed") { + return { type: "proposal", action: "proposal-proposed" }; + } + if (value === "drafted") { + return { type: "review", action: "drafted" }; + } + if (value === "in_review") { + return { type: "review", action: "in-review" }; + } + if (value === "accepted") { + return { type: "review", action: "accepted" }; + } + if (value === "rejected") { + return { type: "review", action: "rejected" }; + } + if (value === "applied") { + return { type: "application", action: "applied" }; + } + if (value === "failed") { + return { type: "application", action: "application-failed" }; + } + return { type: "supersession", action: "superseded" }; +} + +function verificationTransition(value: string): { + type: ActivityHistoryEventType; + action: ActivityHistoryAction; +} { + if (value === "pending") { + return { type: "verification", action: "verification-pending" }; + } + if (value === "due") { + return { type: "verification", action: "verification-due" }; + } + if (value === "overdue") { + return { type: "verification", action: "verification-overdue" }; + } + if (value === "improved") { + return { type: "effectiveness", action: "improved" }; + } + if (value === "unchanged") { + return { type: "effectiveness", action: "unchanged" }; + } + if (value === "regressed") { + return { type: "regression", action: "regressed" }; + } + if (value === "inconclusive") { + return { type: "effectiveness", action: "inconclusive" }; + } + return { type: "verification", action: "verification-unscheduled" }; +} + +function stateTransition(value: string): { + type: ActivityHistoryEventType; + action: ActivityHistoryAction; +} { + if (value === "resolved") { + return { type: "resolution", action: "resolved" }; + } + if (value === "regressed") { + return { type: "regression", action: "regressed" }; + } + if (value === "deferred") { + return { type: "disposition", action: "defer" }; + } + return { type: "observation", action: "state-changed" }; +} + +function eventFingerprints(item: ActivityItem): { + evidence: string; + title: string; + linkedWork: string; +} { + return { + evidence: sha256( + JSON.stringify({ + count: item.evidence.count, + writebacks: [...item.evidence.writebackIds].sort(), + sources: [...item.sourceLabels].sort(), + }) + ), + title: sha256(boundedText(item.title)), + linkedWork: sha256(JSON.stringify(item.linkedWork.map(boundedText).sort())), + }; +} + +function appendItemEvents(args: { + run: ActivityHistoryRun; + item: ActivityItem; + prior?: ActivityHistoryHead; +}): { events: ActivityHistoryEvent[]; head: ActivityHistoryHead } { + const resource = resourceIdentity(args.run.scopeId, args.item); + const context = eventContext(args.item); + const fingerprints = eventFingerprints(args.item); + const correlated = + args.item.kind === "signal" && + (args.item.evidence.writebackIds.length > 1 || + args.item.sourceLabels.length > 1); + const mergedResourceId = args.item.linkedWork + .filter((entry) => entry.startsWith("merged:family:")) + .map((entry) => entry.slice("merged:family:".length)) + .filter(Boolean) + .map((familyId) => + opaqueIdentity("resource", args.run.scopeId, `signal:${familyId}`) + )[0]; + const events: ActivityHistoryEvent[] = []; + const push = ( + type: ActivityHistoryEventType, + action: ActivityHistoryAction, + transition?: ActivityHistoryEvent["transition"], + relatedResourceIds?: string[] + ) => { + events.push( + historyEvent({ + run: args.run, + type, + action, + resource, + relatedResourceIds, + transition, + context, + }) + ); + }; + + if (args.prior) { + if (args.prior.evidenceFingerprint !== fingerprints.evidence) { + push("observation", "repeated", { + field: "evidence", + from: String(args.prior.evidenceCount), + to: String(args.item.evidence.count), + }); + } + if (args.prior.titleFingerprint !== fingerprints.title) { + push("observation", "metadata-updated", { + field: "metadata", + to: "updated", + }); + } + if (args.prior.linkedWorkFingerprint !== fingerprints.linkedWork) { + push("observation", "linked-work-updated", { + field: "linked-work", + to: "updated", + }); + } + if (args.prior.state !== args.item.state) { + const mapped = mergedResourceId + ? { type: "supersession" as const, action: "superseded" as const } + : stateTransition(args.item.state); + push( + mapped.type, + mapped.action, + { + field: "state", + from: args.prior.state, + to: args.item.state, + }, + mergedResourceId ? [mergedResourceId] : undefined + ); + } + } else { + push("discovery", "discovered", { + field: "state", + to: args.item.state, + }); + if (mergedResourceId) { + push( + "supersession", + "superseded", + { field: "state", to: args.item.state }, + [mergedResourceId] + ); + } + } + + if (args.prior?.correlated !== correlated && (correlated || args.prior)) { + push("correlation", correlated ? "correlated" : "correlation-split", { + field: "correlation", + from: args.prior?.correlated ? "correlated" : "uncorrelated", + to: correlated ? "correlated" : "uncorrelated", + }); + } + + const disposition = args.item.decision.disposition; + if (disposition && args.prior?.disposition !== disposition) { + push("disposition", dispositionAction(disposition), { + field: "disposition", + from: args.prior?.disposition, + to: disposition, + }); + } + + const proposalStatus = args.item.decision.proposalStatus; + if (proposalStatus && args.prior?.proposalStatus !== proposalStatus) { + const mapped = proposalTransition(proposalStatus); + push(mapped.type, mapped.action, { + field: "proposal-status", + from: args.prior?.proposalStatus, + to: proposalStatus, + }); + } + + const verification = args.item.verification?.state; + if (verification && args.prior?.verification !== verification) { + const mapped = verificationTransition(verification); + push(mapped.type, mapped.action, { + field: "verification", + from: args.prior?.verification, + to: verification, + }); + if ( + ["improved", "unchanged", "regressed", "inconclusive"].includes( + verification + ) + ) { + push("verification", "verified", { + field: "verification", + from: args.prior?.verification, + to: verification, + }); + } + } + + const firstRecordedAt = args.prior?.firstRecordedAt ?? args.run.recordedAt; + return { + events, + head: { + resource, + state: args.item.state, + disposition, + proposalStatus, + verification, + evidenceFingerprint: fingerprints.evidence, + evidenceCount: args.item.evidence.count, + correlated, + titleFingerprint: fingerprints.title, + linkedWorkFingerprint: fingerprints.linkedWork, + firstRecordedAt, + lastRecordedAt: args.run.recordedAt, + eventCount: (args.prior?.eventCount ?? 0) + events.length, + }, + }; +} + +function buildRun(args: { + report: EvolutionLoopReport; + review?: ReconciliationReview | null; + configRevision: number; + scopeId: string; +}): ActivityHistoryRun { + const degraded = args.report.coverage.filter( + (entry) => entry.state === "unavailable" || entry.state === "stale" + ).length; + return { + id: + portableIdentifier(args.report.runId) ?? + `run:${sha256(args.report.runId).slice(0, 24)}`, + recordedAt: args.report.generatedAt, + scopeId: args.scopeId, + scope: args.report.scope, + trigger: args.report.trigger, + status: args.report.status, + revision: args.report.generationAfter, + configRevision: args.configRevision, + reviewId: portableIdentifier(args.report.reviewId), + window: args.review + ? { + since: args.review.window.since, + until: args.review.window.until, + } + : undefined, + coverage: { + complete: args.report.coverageComplete, + checked: args.report.coverage.length - degraded, + degraded, + }, + }; +} + +function emptyManifest(args: { + run: ActivityHistoryRun; + retentionDays: number; + retentionEvents: number; + retentionHeads: number; +}): ActivityHistoryManifest { + return { + version: 1, + scopeId: args.run.scopeId, + scope: args.run.scope, + createdAt: args.run.recordedAt, + updatedAt: args.run.recordedAt, + migration: { + strategy: "no-backfill", + snapshotOnlyBefore: args.run.recordedAt, + }, + retention: { + maxAgeDays: args.retentionDays, + maxEvents: args.retentionEvents, + maxHeads: args.retentionHeads, + prunedEvents: 0, + prunedSegments: 0, + prunedHeads: 0, + }, + segments: [], + heads: {}, + }; +} + +function parseHead(value: unknown): ActivityHistoryHead | null { + if ( + !( + isRecord(value) && + isRecord(value.resource) && + isPortableIdentifier(value.resource.id) && + ["signal", "proposal", "coverage"].includes(String(value.resource.kind)) + ) || + typeof value.state !== "string" || + (value.resource.itemId !== undefined && + !isPortableIdentifier(value.resource.itemId)) || + (value.resource.familyId !== undefined && + !isPortableIdentifier(value.resource.familyId)) || + (value.resource.proposalId !== undefined && + !isPortableIdentifier(value.resource.proposalId)) || + (value.disposition !== undefined && + typeof value.disposition !== "string") || + (value.proposalStatus !== undefined && + typeof value.proposalStatus !== "string") || + (value.verification !== undefined && + typeof value.verification !== "string") || + typeof value.evidenceFingerprint !== "string" || + !isNonNegativeInteger(value.evidenceCount) || + typeof value.correlated !== "boolean" || + typeof value.titleFingerprint !== "string" || + typeof value.linkedWorkFingerprint !== "string" || + !isIsoDate(value.firstRecordedAt) || + !isIsoDate(value.lastRecordedAt) || + !isNonNegativeInteger(value.eventCount) + ) { + return null; + } + return value as unknown as ActivityHistoryHead; +} + +function parseManifest(value: unknown): ActivityHistoryManifest { + if ( + !isRecord(value) || + value.version !== HISTORY_VERSION || + typeof value.scopeId !== "string" || + (value.scope !== "global" && value.scope !== "project") || + !isIsoDate(value.createdAt) || + !isIsoDate(value.updatedAt) || + !isRecord(value.migration) || + value.migration.strategy !== "no-backfill" || + !isIsoDate(value.migration.snapshotOnlyBefore) || + !isRecord(value.retention) || + !Number.isSafeInteger(value.retention.maxAgeDays) || + Number(value.retention.maxAgeDays) < 1 || + !Number.isSafeInteger(value.retention.maxEvents) || + Number(value.retention.maxEvents) < 1 || + !Number.isSafeInteger(value.retention.maxHeads) || + Number(value.retention.maxHeads) < 1 || + !isNonNegativeInteger(value.retention.prunedEvents) || + !isNonNegativeInteger(value.retention.prunedSegments) || + !isNonNegativeInteger(value.retention.prunedHeads) || + (value.retention.prunedBefore !== undefined && + !isIsoDate(value.retention.prunedBefore)) || + !Array.isArray(value.segments) || + !isRecord(value.heads) + ) { + throw new Error("Unsupported or malformed activity history manifest"); + } + const segments: ActivityHistorySegmentRecord[] = value.segments.map( + (entry) => { + if ( + !(isRecord(entry) && isPortableIdentifier(entry.id)) || + typeof entry.file !== "string" || + basename(entry.file) !== entry.file || + !isIsoDate(entry.recordedAt) || + !isNonNegativeInteger(entry.eventCount) || + typeof entry.checksum !== "string" || + !SHA256_RE.test(entry.checksum) + ) { + throw new Error("Malformed activity history segment record"); + } + return entry as unknown as ActivityHistorySegmentRecord; + } + ); + const heads: Record = {}; + for (const [key, entry] of Object.entries(value.heads)) { + const parsed = parseHead(entry); + if (!parsed || parsed.resource.id !== key) { + throw new Error("Malformed activity history lineage head"); + } + heads[key] = parsed; + } + if ( + new Set(segments.map((entry) => entry.id)).size !== segments.length || + new Set(segments.map((entry) => entry.file)).size !== segments.length + ) { + throw new Error("Duplicate activity history segment record"); + } + return { + ...(value as unknown as ActivityHistoryManifest), + segments, + heads, + }; +} + +async function readBoundedJson( + pathValue: string, + maxBytes: number +): Promise { + const info = await lstat(pathValue); + if (!info.isFile() || info.isSymbolicLink() || info.size > maxBytes) { + throw new Error("Activity history file is not a bounded regular file"); + } + return JSON.parse(await readFile(pathValue, "utf8")) as unknown; +} + +async function fileExists(pathValue: string): Promise { + try { + return (await lstat(pathValue)).isFile(); + } catch { + return false; + } +} + +async function atomicWrite(pathValue: string, body: string): Promise { + await mkdir(dirname(pathValue), { recursive: true }); + const temporary = `${pathValue}.tmp-${process.pid}-${randomUUID()}`; + await writeFile(temporary, body, "utf8"); + await rename(temporary, pathValue); +} + +function retainedManifest(args: { + manifest: ActivityHistoryManifest; + now: string; +}): { + manifest: ActivityHistoryManifest; + removed: ActivityHistorySegmentRecord[]; +} { + const cutoff = + Date.parse(args.now) - + args.manifest.retention.maxAgeDays * 24 * 60 * 60 * 1000; + const sorted = [...args.manifest.segments].sort( + (left, right) => + dateValue(left.recordedAt) - dateValue(right.recordedAt) || + left.id.localeCompare(right.id) + ); + const retained = sorted.filter( + (segment, index) => + index === sorted.length - 1 || Date.parse(segment.recordedAt) >= cutoff + ); + while ( + retained.length > 1 && + retained.reduce((total, segment) => total + segment.eventCount, 0) > + args.manifest.retention.maxEvents + ) { + retained.shift(); + } + const retainedIds = new Set(retained.map((segment) => segment.id)); + const removed = sorted.filter((segment) => !retainedIds.has(segment.id)); + const sortedHeads = Object.entries(args.manifest.heads).sort( + ([leftId, left], [rightId, right]) => + dateValue(right.lastRecordedAt) - dateValue(left.lastRecordedAt) || + leftId.localeCompare(rightId) + ); + const retainedHeads = sortedHeads.slice(0, args.manifest.retention.maxHeads); + const prunedHeads = sortedHeads.length - retainedHeads.length; + const prunedBefore = removed.reduce( + (latest, segment) => + !latest || dateValue(segment.recordedAt) > dateValue(latest) + ? segment.recordedAt + : latest, + args.manifest.retention.prunedBefore + ); + return { + manifest: { + ...args.manifest, + segments: retained, + heads: Object.fromEntries(retainedHeads), + retention: { + ...args.manifest.retention, + prunedEvents: + args.manifest.retention.prunedEvents + + removed.reduce((total, segment) => total + segment.eventCount, 0), + prunedSegments: args.manifest.retention.prunedSegments + removed.length, + prunedHeads: args.manifest.retention.prunedHeads + prunedHeads, + prunedBefore, + }, + }, + removed, + }; +} + +export async function appendActivityHistory(args: { + homeDir: string; + rootDir: string; + report: EvolutionLoopReport; + review?: ReconciliationReview | null; + configRevision: number; + retention?: { maxAgeDays?: number; maxEvents?: number; maxHeads?: number }; +}): Promise<{ appended: boolean; eventCount: number; prunedEvents: number }> { + if (args.report.status === "preview") { + return { appended: false, eventCount: 0, prunedEvents: 0 }; + } + const retentionDays = args.retention?.maxAgeDays ?? DEFAULT_RETENTION_DAYS; + const retentionEvents = args.retention?.maxEvents ?? DEFAULT_RETENTION_EVENTS; + const retentionHeads = args.retention?.maxHeads ?? DEFAULT_RETENTION_HEADS; + if ( + !Number.isSafeInteger(retentionDays) || + retentionDays < 1 || + !Number.isSafeInteger(retentionEvents) || + retentionEvents < 1 || + !Number.isSafeInteger(retentionHeads) || + retentionHeads < 1 + ) { + throw new Error( + "Activity history retention limits must be positive integers" + ); + } + const scopeId = + args.report.scope === "global" + ? "global" + : machineStateProjectScopeId( + machineStateProjectKey(args.rootDir, args.homeDir) + ); + const run = buildRun({ + report: args.report, + review: args.review, + configRevision: args.configRevision, + scopeId, + }); + const manifestPath = facultAiActivityHistoryManifestPath( + args.homeDir, + args.rootDir + ); + const segmentDir = facultAiActivityHistorySegmentDir( + args.homeDir, + args.rootDir + ); + let manifest = (await fileExists(manifestPath)) + ? parseManifest(await readBoundedJson(manifestPath, MAX_MANIFEST_BYTES)) + : emptyManifest({ + run, + retentionDays, + retentionEvents, + retentionHeads, + }); + if (manifest.scopeId !== scopeId || manifest.scope !== args.report.scope) { + throw new Error( + "Activity history manifest scope does not match the current run" + ); + } + if (manifest.segments.some((segment) => segment.id === run.id)) { + return { + appended: false, + eventCount: 0, + prunedEvents: manifest.retention.prunedEvents, + }; + } + + const events: ActivityHistoryEvent[] = [ + historyEvent({ + run, + type: "run", + action: "run-recorded", + }), + ]; + const heads = { ...manifest.heads }; + for (const item of args.report.activity?.items ?? []) { + const identity = resourceIdentity(scopeId, item); + const result = appendItemEvents({ run, item, prior: heads[identity.id] }); + events.push(...result.events); + heads[identity.id] = result.head; + } + const segment: ActivityHistorySegment = { + version: 1, + run, + events, + }; + const body = `${JSON.stringify(segment, null, 2)}\n`; + if (Buffer.byteLength(body) > MAX_SEGMENT_BYTES) { + throw new Error( + "Activity history segment exceeds the bounded segment size" + ); + } + const segmentFile = `segment-${sha256(`${scopeId}\n${run.id}`).slice(0, 24)}.json`; + const segmentPath = join(segmentDir, segmentFile); + await mkdir(segmentDir, { recursive: true }); + try { + await writeFile(segmentPath, body, { encoding: "utf8", flag: "wx" }); + } catch (error) { + if ( + !(error instanceof Error && "code" in error) || + (error as NodeJS.ErrnoException).code !== "EEXIST" + ) { + throw error; + } + const existing = await readFile(segmentPath, "utf8"); + if (sha256(existing) !== sha256(body)) { + throw new Error( + "Activity history run identity conflicts with existing content" + ); + } + } + manifest = { + ...manifest, + updatedAt: run.recordedAt, + retention: { + ...manifest.retention, + maxAgeDays: args.retention?.maxAgeDays ?? manifest.retention.maxAgeDays, + maxEvents: args.retention?.maxEvents ?? manifest.retention.maxEvents, + maxHeads: args.retention?.maxHeads ?? manifest.retention.maxHeads, + }, + segments: [ + ...manifest.segments, + { + id: run.id, + file: segmentFile, + recordedAt: run.recordedAt, + eventCount: events.length, + checksum: sha256(body), + }, + ], + heads, + }; + const retained = retainedManifest({ manifest, now: run.recordedAt }); + const manifestBody = `${JSON.stringify(retained.manifest, null, 2)}\n`; + if (Buffer.byteLength(manifestBody) > MAX_MANIFEST_BYTES) { + throw new Error("Activity history manifest exceeds its bounded size"); + } + await atomicWrite(manifestPath, manifestBody); + await Promise.all( + retained.removed.map(async (entry) => { + await rm(join(segmentDir, entry.file), { force: true }); + }) + ); + return { + appended: true, + eventCount: events.length, + prunedEvents: retained.manifest.retention.prunedEvents, + }; +} + +function parseResource( + value: unknown +): ActivityHistoryResourceIdentity | undefined | null { + if (value === undefined) { + return undefined; + } + if ( + !( + isRecord(value) && + isPortableIdentifier(value.id) && + ["signal", "proposal", "coverage"].includes(String(value.kind)) + ) || + (value.itemId !== undefined && !isPortableIdentifier(value.itemId)) || + (value.familyId !== undefined && !isPortableIdentifier(value.familyId)) || + (value.proposalId !== undefined && !isPortableIdentifier(value.proposalId)) + ) { + return null; + } + return value as unknown as ActivityHistoryResourceIdentity; +} + +function parseTransition( + value: unknown +): ActivityHistoryEvent["transition"] | null { + if (value === undefined) { + return undefined; + } + if ( + !( + isRecord(value) && + HISTORY_TRANSITION_FIELDS.has( + value.field as NonNullable["field"] + ) + ) || + (value.from !== undefined && typeof value.from !== "string") || + typeof value.to !== "string" + ) { + return null; + } + return value as unknown as ActivityHistoryEvent["transition"]; +} + +function parseContext(value: unknown): ActivityHistoryEvent["context"] | null { + if (value === undefined) { + return undefined; + } + if ( + !isRecord(value) || + typeof value.title !== "string" || + !isNonNegativeInteger(value.evidenceCount) || + !Array.isArray(value.sourceLabels) || + value.sourceLabels.length > 10 || + value.sourceLabels.some((entry) => typeof entry !== "string") || + (value.rationale !== undefined && typeof value.rationale !== "string") || + (value.target !== undefined && typeof value.target !== "string") || + !Array.isArray(value.linkedWork) || + value.linkedWork.length > 10 || + value.linkedWork.some((entry) => typeof entry !== "string") || + !Array.isArray(value.links) || + value.links.length > 5 || + value.links.some( + (entry) => + !isRecord(entry) || + typeof entry.label !== "string" || + typeof entry.url !== "string" + ) + ) { + return null; + } + return value as unknown as ActivityHistoryEvent["context"]; +} + +function parseEvent(value: unknown): ActivityHistoryEvent | null { + if ( + !isRecord(value) || + value.version !== HISTORY_VERSION || + !isPortableIdentifier(value.id) || + !isIsoDate(value.recordedAt) || + !HISTORY_EVENT_TYPES.has(value.type as ActivityHistoryEventType) || + !HISTORY_ACTIONS.has(value.action as ActivityHistoryAction) || + !isPortableIdentifier(value.scopeId) || + !isPortableIdentifier(value.runId) || + (value.relatedResourceIds !== undefined && + (!Array.isArray(value.relatedResourceIds) || + value.relatedResourceIds.length > 10 || + value.relatedResourceIds.some((entry) => !isPortableIdentifier(entry)))) + ) { + return null; + } + const resource = parseResource(value.resource); + const transition = parseTransition(value.transition); + const context = parseContext(value.context); + if (resource === null || transition === null || context === null) { + return null; + } + return { + ...(value as unknown as ActivityHistoryEvent), + resource, + transition, + context, + }; +} + +function parseRun(value: unknown): ActivityHistoryRun | null { + if ( + !( + isRecord(value) && + isPortableIdentifier(value.id) && + isIsoDate(value.recordedAt) && + isPortableIdentifier(value.scopeId) + ) || + (value.scope !== "global" && value.scope !== "project") || + (value.trigger !== "manual" && value.trigger !== "scheduled") || + !["complete", "degraded", "failed"].includes(String(value.status)) || + !isNonNegativeInteger(value.revision) || + !isNonNegativeInteger(value.configRevision) || + (value.reviewId !== undefined && !isPortableIdentifier(value.reviewId)) || + (value.window !== undefined && + (!( + isRecord(value.window) && + isIsoDate(value.window.since) && + isIsoDate(value.window.until) + ) || + dateValue(value.window.since) > dateValue(value.window.until))) || + !isRecord(value.coverage) || + typeof value.coverage.complete !== "boolean" || + !isNonNegativeInteger(value.coverage.checked) || + !isNonNegativeInteger(value.coverage.degraded) + ) { + return null; + } + return value as unknown as ActivityHistoryRun; +} + +function parseSegment(value: unknown): ActivityHistorySegment | null { + if ( + !isRecord(value) || + value.version !== HISTORY_VERSION || + !Array.isArray(value.events) + ) { + return null; + } + const run = parseRun(value.run); + const events = value.events.map(parseEvent); + if (!run || events.some((event) => !event)) { + return null; + } + return { + version: 1, + run, + events: events as ActivityHistoryEvent[], + }; +} + +function portableEvent(event: ActivityHistoryEvent): ActivityHistoryEvent { + return { + ...event, + context: event.context + ? { + ...event.context, + title: boundedText(event.context.title), + sourceLabels: event.context.sourceLabels.map(boundedText), + rationale: event.context.rationale + ? boundedText(event.context.rationale) + : undefined, + target: event.context.target + ? boundedText(event.context.target) + : undefined, + linkedWork: event.context.linkedWork.map(boundedText), + links: event.context.links + .map(safeLink) + .filter((entry): entry is { label: string; url: string } => + Boolean(entry) + ), + } + : undefined, + }; +} + +function parseCursor(value: string | undefined): { + recordedAt: string; + id: string; +} | null { + if (!value) { + return null; + } + try { + const parsed = JSON.parse( + Buffer.from(value, "base64url").toString("utf8") + ) as unknown; + if ( + !isRecord(parsed) || + parsed.version !== 1 || + !isIsoDate(parsed.recordedAt) || + typeof parsed.id !== "string" + ) { + throw new Error("invalid cursor"); + } + return { recordedAt: parsed.recordedAt, id: parsed.id }; + } catch { + throw new Error("Invalid activity history cursor"); + } +} + +function encodeCursor(event: ActivityHistoryEvent): string { + return Buffer.from( + JSON.stringify({ version: 1, recordedAt: event.recordedAt, id: event.id }) + ).toString("base64url"); +} + +function beforeCursor( + event: ActivityHistoryEvent, + cursor: ReturnType +): boolean { + if (!cursor) { + return true; + } + return ( + dateValue(event.recordedAt) < dateValue(cursor.recordedAt) || + (dateValue(event.recordedAt) === dateValue(cursor.recordedAt) && + event.id < cursor.id) + ); +} + +function matchesItem( + event: ActivityHistoryEvent, + item: string | undefined +): boolean { + if (!item) { + return true; + } + const resource = event.resource; + return Boolean( + resource && + [ + resource.id, + resource.itemId, + resource.familyId, + resource.proposalId, + ].includes(item) + ); +} + +function matchesFilters( + event: ActivityHistoryEvent, + query: ActivityHistoryQuery, + cursor: ReturnType +): boolean { + return ( + (!query.since || dateValue(event.recordedAt) >= dateValue(query.since)) && + (!query.until || dateValue(event.recordedAt) <= dateValue(query.until)) && + (!query.scopeId || event.scopeId === query.scopeId) && + (!query.eventTypes?.length || query.eventTypes.includes(event.type)) && + matchesItem(event, query.item) && + beforeCursor(event, cursor) + ); +} + +async function readScope(args: { + descriptor: ScopeDescriptor; + query: ActivityHistoryQuery; + cursor: ReturnType; + budget: { remaining: number }; +}): Promise { + if (args.descriptor.omitted) { + return { + descriptor: args.descriptor, + events: [], + runs: [], + state: "omitted", + corruptSegments: 0, + scanLimitReached: false, + }; + } + if (!(await fileExists(args.descriptor.manifestPath))) { + return { + descriptor: args.descriptor, + events: [], + runs: [], + state: args.descriptor.configured ? "snapshot-only" : "unavailable", + detail: args.descriptor.configured + ? "history-not-recorded" + : "loop-not-configured", + corruptSegments: 0, + scanLimitReached: false, + }; + } + let manifest: ActivityHistoryManifest; + try { + manifest = parseManifest( + await readBoundedJson(args.descriptor.manifestPath, MAX_MANIFEST_BYTES) + ); + } catch { + return { + descriptor: args.descriptor, + events: [], + runs: [], + state: "degraded", + detail: "history-manifest-invalid", + corruptSegments: 0, + scanLimitReached: false, + }; + } + if ( + manifest.scopeId !== args.descriptor.id || + manifest.scope !== args.descriptor.scope + ) { + return { + descriptor: args.descriptor, + manifest, + events: [], + runs: [], + state: "degraded", + detail: "history-scope-mismatch", + corruptSegments: 0, + scanLimitReached: false, + }; + } + const events: ActivityHistoryEvent[] = []; + const runs: ActivityHistoryRun[] = []; + let corruptSegments = 0; + let scanLimitReached = false; + const segments = [...manifest.segments].sort( + (left, right) => + dateValue(right.recordedAt) - dateValue(left.recordedAt) || + right.id.localeCompare(left.id) + ); + for (const record of segments) { + if (args.budget.remaining <= 0) { + scanLimitReached = true; + break; + } + try { + const pathValue = join(args.descriptor.segmentDir, record.file); + const raw = await readFile(pathValue, "utf8"); + if ( + Buffer.byteLength(raw) > MAX_SEGMENT_BYTES || + sha256(raw) !== record.checksum + ) { + throw new Error("invalid segment bounds or checksum"); + } + const segment = parseSegment(JSON.parse(raw) as unknown); + if (!segment || segment.run.id !== record.id) { + throw new Error("invalid segment schema"); + } + runs.push(segment.run); + for (const event of segment.events) { + if (args.budget.remaining <= 0) { + scanLimitReached = true; + break; + } + args.budget.remaining -= 1; + if (matchesFilters(event, args.query, args.cursor)) { + events.push(portableEvent(event)); + } + } + } catch { + corruptSegments += 1; + } + } + return { + descriptor: args.descriptor, + manifest, + events, + runs, + state: corruptSegments || scanLimitReached ? "degraded" : "available", + detail: corruptSegments ? "history-segment-invalid" : undefined, + corruptSegments, + scanLimitReached, + }; +} + +async function isConfiguredLoop(pathValue: string): Promise { + try { + const config = await readBoundedJson(pathValue, 100_000); + return ( + isRecord(config) && + (config.scope === "global" || config.scope === "project") + ); + } catch { + return false; + } +} + +async function scopeDescriptors(query: ActivityHistoryQuery): Promise<{ + descriptors: ScopeDescriptor[]; + discoveryTruncated: boolean; +}> { + if (query.scope === "global" || query.scope === "project") { + const scopeId = + query.scope === "global" + ? "global" + : machineStateProjectScopeId( + machineStateProjectKey(query.rootDir, query.homeDir) + ); + return { + descriptors: [ + { + id: scopeId, + scope: query.scope, + configured: await isConfiguredLoop( + facultAiEvolutionLoopConfigPath(query.homeDir, query.rootDir) + ), + manifestPath: facultAiActivityHistoryManifestPath( + query.homeDir, + query.rootDir + ), + segmentDir: facultAiActivityHistorySegmentDir( + query.homeDir, + query.rootDir + ), + omitted: false, + }, + ], + discoveryTruncated: false, + }; + } + + const descriptors: ScopeDescriptor[] = []; + const globalManifest = facultAiActivityHistoryManifestPath( + query.homeDir, + query.rootDir + ); + const globalConfigured = await isConfiguredLoop( + facultAiEvolutionLoopConfigPath(query.homeDir, query.rootDir) + ); + if (globalConfigured || (await fileExists(globalManifest))) { + descriptors.push({ + id: "global", + scope: "global", + configured: globalConfigured, + manifestPath: globalManifest, + segmentDir: facultAiActivityHistorySegmentDir( + query.homeDir, + query.rootDir + ), + omitted: false, + }); + } + const projectsDir = join(facultLocalStateRoot(query.homeDir), "projects"); + let entries: Dirent[] = []; + try { + entries = await readdir(projectsDir, { withFileTypes: true }); + } catch { + return { descriptors, discoveryTruncated: false }; + } + const directories = entries + .filter((entry) => entry.isDirectory()) + .sort((left, right) => left.name.localeCompare(right.name)); + const discoveryTruncated = directories.length > MAX_DISCOVERED_PROJECTS; + for (const entry of directories.slice(0, MAX_DISCOVERED_PROJECTS)) { + const loopDir = join( + projectsDir, + entry.name, + "ai", + "project", + "evolution", + "loop" + ); + const manifestPath = join(loopDir, "history", "manifest.json"); + const configured = await isConfiguredLoop(join(loopDir, "config.json")); + if (!(configured || (await fileExists(manifestPath)))) { + continue; + } + descriptors.push({ + id: machineStateProjectScopeId(entry.name), + scope: "project", + configured, + manifestPath, + segmentDir: join(loopDir, "history", "segments"), + omitted: false, + }); + } + return { + descriptors: descriptors.map((descriptor, index) => ({ + ...descriptor, + omitted: index >= MAX_QUERY_SCOPES, + })), + discoveryTruncated, + }; +} + +function validateQuery(query: ActivityHistoryQuery): number { + for (const [name, value] of [ + ["since", query.since], + ["until", query.until], + ] as const) { + if (value && !isIsoDate(value)) { + throw new Error(`Activity history --${name} must be an ISO-8601 date`); + } + } + if ( + query.since && + query.until && + dateValue(query.since) > dateValue(query.until) + ) { + throw new Error("Activity history --since must not be after --until"); + } + const limit = query.limit ?? DEFAULT_QUERY_LIMIT; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_QUERY_LIMIT) { + throw new Error( + `Activity history --limit must be between 1 and ${MAX_QUERY_LIMIT}` + ); + } + return limit; +} + +function headMatches(head: ActivityHistoryHead, item: string): boolean { + return [ + head.resource.id, + head.resource.itemId, + head.resource.familyId, + head.resource.proposalId, + ].includes(item); +} + +export async function queryActivityHistory( + query: ActivityHistoryQuery +): Promise { + const limit = validateQuery(query); + const cursor = parseCursor(query.cursor); + const discovered = await scopeDescriptors(query); + const budget = { remaining: MAX_QUERY_SCANNED_EVENTS }; + const reads: ScopeRead[] = []; + for (const descriptor of discovered.descriptors) { + if (query.scopeId && descriptor.id !== query.scopeId) { + continue; + } + reads.push(await readScope({ descriptor, query, cursor, budget })); + } + const sortedEvents = reads + .flatMap((read) => read.events) + .sort( + (left, right) => + dateValue(right.recordedAt) - dateValue(left.recordedAt) || + right.id.localeCompare(left.id) + ); + const pageEvents = sortedEvents.slice(0, limit); + const hasMore = sortedEvents.length > limit; + const returnedRunIds = new Set(pageEvents.map((event) => event.runId)); + const runs = reads + .flatMap((read) => read.runs) + .filter((run) => returnedRunIds.has(run.id)) + .filter( + (run, index, values) => + values.findIndex( + (candidate) => + candidate.id === run.id && candidate.scopeId === run.scopeId + ) === index + ) + .sort( + (left, right) => dateValue(right.recordedAt) - dateValue(left.recordedAt) + ); + const scopes = reads.map((read) => { + const manifest = read.manifest; + return { + id: read.descriptor.id, + scope: read.descriptor.scope, + state: read.state, + historyStart: manifest?.segments[0]?.recordedAt, + snapshotOnlyBefore: manifest?.migration.snapshotOnlyBefore, + prunedBefore: manifest?.retention.prunedBefore, + prunedEvents: manifest?.retention.prunedEvents ?? 0, + prunedHeads: manifest?.retention.prunedHeads ?? 0, + corruptSegments: read.corruptSegments, + detail: read.detail, + }; + }); + const omittedScopes = discovered.descriptors.filter( + (entry) => entry.omitted + ).length; + const scanLimitReached = reads.some((read) => read.scanLimitReached); + const requestedWindowIsComplete = reads.every((read) => { + const manifest = read.manifest; + if (!manifest || read.state !== "available") { + return false; + } + if (query.item && manifest.retention.prunedHeads > 0) { + return false; + } + const since = query.since; + if ( + !since || + dateValue(since) < dateValue(manifest.migration.snapshotOnlyBefore) + ) { + return false; + } + return !( + manifest.retention.prunedBefore && + dateValue(since) <= dateValue(manifest.retention.prunedBefore) + ); + }); + const complete = + reads.length > 0 && + requestedWindowIsComplete && + !omittedScopes && + !discovered.discoveryTruncated && + !scanLimitReached; + const unavailable = + reads.length === 0 || reads.every((read) => read.state === "unavailable"); + const lineageResources = query.item + ? reads.flatMap((read) => + Object.values(read.manifest?.heads ?? {}) + .filter((head) => headMatches(head, query.item!)) + .map((head) => ({ + scopeId: read.descriptor.id, + resource: head.resource, + firstRecordedAt: head.firstRecordedAt, + lastRecordedAt: head.lastRecordedAt, + eventCount: head.eventCount, + current: { + state: head.state, + disposition: head.disposition, + proposalStatus: head.proposalStatus, + verification: head.verification, + }, + })) + ) + : []; + return { + version: 1, + kind: "activity-history", + mode: "timeline", + filters: { + scope: query.scope, + since: query.since, + until: query.until, + item: query.item, + scopeId: query.scopeId, + eventTypes: query.eventTypes ?? [], + }, + capabilities: { + externalMutation: false, + export: false, + rawPayloads: false, + }, + coverage: { + state: unavailable ? "unavailable" : complete ? "complete" : "partial", + complete, + configuredScopes: reads.filter((read) => read.descriptor.configured) + .length, + reportingScopes: reads.filter((read) => read.state === "available") + .length, + degradedScopes: reads.filter((read) => read.state === "degraded").length, + snapshotOnlyScopes: reads.filter((read) => read.state === "snapshot-only") + .length, + scopes, + }, + retention: { + defaultMaxAgeDays: DEFAULT_RETENTION_DAYS, + defaultMaxEventsPerScope: DEFAULT_RETENTION_EVENTS, + defaultMaxLineageHeadsPerScope: DEFAULT_RETENTION_HEADS, + migration: "no-backfill-from-snapshots-or-journals", + }, + events: pageEvents, + runs, + lineage: query.item + ? { + query: query.item, + ambiguous: lineageResources.length > 1, + resources: lineageResources, + } + : undefined, + page: { + limit, + nextCursor: + hasMore && pageEvents.length + ? encodeCursor(pageEvents.at(-1)!) + : undefined, + }, + truncation: { + truncated: + Boolean(omittedScopes) || + discovered.discoveryTruncated || + scanLimitReached, + omittedScopes, + scanLimitReached, + }, + }; +} + +export function renderActivityHistory( + result: ActivityHistoryQueryResult +): string { + const lines = [ + `Activity history · ${result.coverage.state}`, + `Scopes: ${result.coverage.reportingScopes}/${result.coverage.configuredScopes} reporting`, + `Events: ${result.events.length}${result.page.nextCursor ? " · more available" : ""}`, + "", + ]; + if (!result.events.length) { + lines.push( + result.coverage.state === "unavailable" + ? "No history is available for the selected scope." + : "No events matched the selected bounded query." + ); + return lines.join("\n"); + } + for (const event of result.events) { + lines.push( + `- ${event.recordedAt} · ${event.action} · ${event.context?.title ?? event.runId}` + ); + } + return lines.join("\n"); +} diff --git a/src/ai-cli.test.ts b/src/ai-cli.test.ts index f20c70f..2e2eb09 100644 --- a/src/ai-cli.test.ts +++ b/src/ai-cli.test.ts @@ -195,6 +195,40 @@ describe("ai CLI", () => { }); expect(activityText).not.toContain(tempHome); + const historyOut = await captureConsole(async () => { + await aiCommand([ + "loop", + "history", + "--global", + "--since", + "2026-01-01T00:00:00.000Z", + "--limit", + "1", + "--json", + ]); + }); + expect(historyOut.errors).toEqual([]); + const historyText = historyOut.logs.join("\n"); + expect(JSON.parse(historyText)).toMatchObject({ + version: 1, + kind: "activity-history", + mode: "timeline", + filters: { scope: "global" }, + capabilities: { externalMutation: false, export: false }, + events: [{ type: "run", action: "run-recorded", scopeId: "global" }], + page: { limit: 1 }, + }); + expect(historyText).not.toContain(tempHome); + + const allHistoryOut = await captureConsole(async () => { + await aiCommand(["loop", "history", "--json"]); + }); + expect(allHistoryOut.errors).toEqual([]); + expect(JSON.parse(allHistoryOut.logs.join("\n"))).toMatchObject({ + filters: { scope: "all" }, + coverage: { configuredScopes: 1 }, + }); + const allActivityOut = await captureConsole(async () => { await aiCommand(["loop", "activity", "--json"]); }); diff --git a/src/ai.ts b/src/ai.ts index 4dcf39e..793bf33 100644 --- a/src/ai.ts +++ b/src/ai.ts @@ -2809,7 +2809,7 @@ Usage: fclt ai writeback [args...] fclt ai evolve [args...] fclt ai review [args...] - fclt ai loop [args...] + fclt ai loop [args...] `; } @@ -2823,6 +2823,7 @@ Usage: fclt ai loop report [--json] fclt ai loop activity [--all|--global|--project] [--json] fclt ai loop resolve [--json] + fclt ai loop history [--all|--global|--project] [--since ] [--until ] [--item ] [--scope-id ] [--event ] [--limit <1-200>] [--cursor ] [--json] fclt ai loop run [--since ] [--until ] [--source ] [--dry-run] [--scheduled] [--json] The loop keeps a full machine-local review queue and emits a delta for @@ -2931,6 +2932,18 @@ function parseRepeatedFlag(argv: string[], flag: string): string[] { return values; } +function parseIntegerFlag(argv: string[], flag: string): number | undefined { + const raw = parseStringFlag(argv, flag); + if (raw === undefined) { + return undefined; + } + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${flag} requires an integer`); + } + return parsed; +} + async function loopCommand(argv: string[]) { const parsed = parseCliContextArgs(argv); const [sub, ...commandArgs] = parsed.argv; @@ -3123,6 +3136,74 @@ async function loopCommand(argv: string[]) { ); return; } + if (sub === "history") { + const { queryActivityHistory, renderActivityHistory } = await import( + "./activity-history" + ); + const explicitAllScopes = commandArgs.includes("--all"); + const allScopes = explicitAllScopes || parsed.scope === "merged"; + if (explicitAllScopes && parsed.scope !== "merged") { + throw new Error("Conflicting scope flags"); + } + const historyRootDir = allScopes + ? resolveCliContextRoot({ + homeDir, + cwd: process.cwd(), + rootArg: parsed.rootArg, + scope: "global", + }) + : rootDir; + if ( + allScopes && + !explicitAllScopes && + projectRootFromAiRoot(historyRootDir, homeDir) + ) { + throw new Error( + "All-scope history accepts only a global --root; use --project for one project" + ); + } + const eventTypes = parseRepeatedFlag(commandArgs, "--event"); + const allowedEventTypes = new Set([ + "run", + "discovery", + "observation", + "correlation", + "disposition", + "proposal", + "review", + "application", + "verification", + "effectiveness", + "regression", + "supersession", + "resolution", + ]); + const invalidEventType = eventTypes.find( + (value) => !allowedEventTypes.has(value) + ); + if (invalidEventType) { + throw new Error( + `Unknown activity history event type: ${invalidEventType}` + ); + } + const result = await queryActivityHistory({ + homeDir, + rootDir: historyRootDir, + scope: allScopes ? "all" : loopScope, + since: parseStringFlag(commandArgs, "--since"), + until: parseStringFlag(commandArgs, "--until"), + item: parseStringFlag(commandArgs, "--item"), + scopeId: parseStringFlag(commandArgs, "--scope-id"), + eventTypes: + eventTypes as import("./activity-history").ActivityHistoryEventType[], + limit: parseIntegerFlag(commandArgs, "--limit"), + cursor: parseStringFlag(commandArgs, "--cursor"), + }); + console.log( + json ? JSON.stringify(result, null, 2) : renderActivityHistory(result) + ); + return; + } if (sub === "run") { const result = await runEvolutionLoop({ homeDir, diff --git a/src/evolution-loop.ts b/src/evolution-loop.ts index 4f8cddc..ff63955 100644 --- a/src/evolution-loop.ts +++ b/src/evolution-loop.ts @@ -1930,6 +1930,14 @@ async function persistFailedLoopRun(args: { recovery: "Resolve the reported error, inspect the failed report, and rerun the same bounded window", }); + const { appendActivityHistory } = await import("./activity-history"); + await appendActivityHistory({ + homeDir: args.homeDir, + rootDir: args.rootDir, + report, + review: args.review, + configRevision: args.config.generation, + }); return report; } @@ -2198,6 +2206,14 @@ async function runEvolutionLoopScoped(args: { })}\n`, "utf8" ); + const { appendActivityHistory } = await import("./activity-history"); + await appendActivityHistory({ + homeDir: args.homeDir, + rootDir: args.rootDir, + report, + review, + configRevision: config.generation, + }); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/src/paths.test.ts b/src/paths.test.ts index f69819f..f147b76 100644 --- a/src/paths.test.ts +++ b/src/paths.test.ts @@ -3,6 +3,8 @@ import { mkdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { + facultAiActivityHistoryManifestPath, + facultAiActivityHistorySegmentDir, facultAiGraphPath, facultAiIndexPath, facultAiStateDir, @@ -13,6 +15,7 @@ import { facultRootDir, facultRuntimeCacheDir, legacyFacultAiStateDirs, + machineStateProjectScopeId, pathIsInsideOrEqual, projectRootFromAiRoot, } from "./paths"; @@ -119,6 +122,31 @@ describe("paths", () => { ); }); + it("stores activity history in runtime state with stable opaque project scope ids", async () => { + tempHome = await makeTempHome(); + process.env.HOME = tempHome; + + const rootDir = join(tempHome, "work", "repo", ".ai"); + const historyDir = join( + facultMachineStateDir(tempHome, rootDir), + "ai", + "project", + "evolution", + "loop", + "history" + ); + + expect(facultAiActivityHistoryManifestPath(tempHome, rootDir)).toBe( + join(historyDir, "manifest.json") + ); + expect(facultAiActivityHistorySegmentDir(tempHome, rootDir)).toBe( + join(historyDir, "segments") + ); + expect(machineStateProjectScopeId("example-machine-key")).toBe( + "project:4bbe4718dff3fbd0" + ); + }); + it("uses machine-local generated AI state for project roots", async () => { tempHome = await makeTempHome(); process.env.HOME = tempHome; diff --git a/src/paths.ts b/src/paths.ts index bf0e437..c6e04ac 100644 --- a/src/paths.ts +++ b/src/paths.ts @@ -568,6 +568,32 @@ export function facultAiEvolutionLoopReportDir( ); } +export function facultAiActivityHistoryDir( + home: string = defaultHomeDir(), + rootDir?: string +): string { + return join( + facultAiRuntimeScopeDir(home, rootDir), + "evolution", + "loop", + "history" + ); +} + +export function facultAiActivityHistoryManifestPath( + home: string = defaultHomeDir(), + rootDir?: string +): string { + return join(facultAiActivityHistoryDir(home, rootDir), "manifest.json"); +} + +export function facultAiActivityHistorySegmentDir( + home: string = defaultHomeDir(), + rootDir?: string +): string { + return join(facultAiActivityHistoryDir(home, rootDir), "segments"); +} + export function facultAiReconciliationConfigPath( home: string = defaultHomeDir(), rootDir?: string From e6076fc7e69ceb2b860e0de8ca997ff25ab4880f Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 12:10:53 -0400 Subject: [PATCH 02/12] fix: keep history pagination progressing --- src/activity-history.ts | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/activity-history.ts b/src/activity-history.ts index 9f2e293..c4e2cd7 100644 --- a/src/activity-history.ts +++ b/src/activity-history.ts @@ -1513,6 +1513,14 @@ async function readScope(args: { right.id.localeCompare(left.id) ); for (const record of segments) { + const recordedAt = dateValue(record.recordedAt); + if ( + (args.query.since && recordedAt < dateValue(args.query.since)) || + (args.query.until && recordedAt > dateValue(args.query.until)) || + (args.cursor && recordedAt > dateValue(args.cursor.recordedAt)) + ) { + continue; + } if (args.budget.remaining <= 0) { scanLimitReached = true; break; @@ -1707,13 +1715,25 @@ export async function queryActivityHistory( const limit = validateQuery(query); const cursor = parseCursor(query.cursor); const discovered = await scopeDescriptors(query); - const budget = { remaining: MAX_QUERY_SCANNED_EVENTS }; + const selectedDescriptors = discovered.descriptors.filter( + (descriptor) => !query.scopeId || descriptor.id === query.scopeId + ); + const perScopeBudget = Math.max( + 1, + Math.floor( + MAX_QUERY_SCANNED_EVENTS / Math.max(selectedDescriptors.length, 1) + ) + ); const reads: ScopeRead[] = []; - for (const descriptor of discovered.descriptors) { - if (query.scopeId && descriptor.id !== query.scopeId) { - continue; - } - reads.push(await readScope({ descriptor, query, cursor, budget })); + for (const descriptor of selectedDescriptors) { + reads.push( + await readScope({ + descriptor, + query, + cursor, + budget: { remaining: perScopeBudget }, + }) + ); } const sortedEvents = reads .flatMap((read) => read.events) From 4a53d4044fa3e5ba344773be0a4426427a7fcbb8 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 12:18:44 -0400 Subject: [PATCH 03/12] fix: honor targeted history scopes --- src/activity-history.test.ts | 19 +++++++++++++++++++ src/activity-history.ts | 17 ++++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/activity-history.test.ts b/src/activity-history.test.ts index ee6eab7..fe8dd1b 100644 --- a/src/activity-history.test.ts +++ b/src/activity-history.test.ts @@ -798,5 +798,24 @@ describe("activity history", () => { expect(aggregate.truncation.truncated).toBe(true); expect(aggregate.truncation.omittedScopes).toBe(2); expect(aggregate.coverage.complete).toBe(false); + + const omittedScope = aggregate.coverage.scopes.find( + (scope) => scope.state === "omitted" + ); + expect(omittedScope).toBeDefined(); + const targeted = await queryActivityHistory({ + homeDir: home, + rootDir: globalRoot, + scope: "all", + scopeId: omittedScope!.id, + since: "2026-07-02T00:00:00.000Z", + eventTypes: ["run"], + }); + expect(targeted.events).toHaveLength(1); + expect(targeted.coverage.scopes[0]?.state).toBe("available"); + expect(targeted.truncation).toMatchObject({ + truncated: false, + omittedScopes: 0, + }); }); }); diff --git a/src/activity-history.ts b/src/activity-history.ts index c4e2cd7..84ed7ad 100644 --- a/src/activity-history.ts +++ b/src/activity-history.ts @@ -1715,14 +1715,17 @@ export async function queryActivityHistory( const limit = validateQuery(query); const cursor = parseCursor(query.cursor); const discovered = await scopeDescriptors(query); - const selectedDescriptors = discovered.descriptors.filter( - (descriptor) => !query.scopeId || descriptor.id === query.scopeId - ); + const selectedDescriptors = discovered.descriptors + .filter((descriptor) => !query.scopeId || descriptor.id === query.scopeId) + .map((descriptor) => + query.scopeId ? { ...descriptor, omitted: false } : descriptor + ); + const readableScopeCount = selectedDescriptors.filter( + (descriptor) => !descriptor.omitted + ).length; const perScopeBudget = Math.max( 1, - Math.floor( - MAX_QUERY_SCANNED_EVENTS / Math.max(selectedDescriptors.length, 1) - ) + Math.floor(MAX_QUERY_SCANNED_EVENTS / Math.max(readableScopeCount, 1)) ); const reads: ScopeRead[] = []; for (const descriptor of selectedDescriptors) { @@ -1773,7 +1776,7 @@ export async function queryActivityHistory( detail: read.detail, }; }); - const omittedScopes = discovered.descriptors.filter( + const omittedScopes = selectedDescriptors.filter( (entry) => entry.omitted ).length; const scanLimitReached = reads.some((read) => read.scanLimitReached); From d603bc20aac1b0dc719e68b7562768a919b6a55b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 12:26:46 -0400 Subject: [PATCH 04/12] fix: scope history run metadata --- src/activity-history.test.ts | 17 +++++++++++++++++ src/activity-history.ts | 6 ++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/activity-history.test.ts b/src/activity-history.test.ts index fe8dd1b..c21a4fd 100644 --- a/src/activity-history.test.ts +++ b/src/activity-history.test.ts @@ -473,6 +473,23 @@ describe("activity history", () => { expect( new Set(result.lineage?.resources.map((entry) => entry.resource.id)).size ).toBe(2); + expect(result.runs).toHaveLength(2); + expect(new Set(result.runs.map((run) => run.scopeId)).size).toBe(2); + expect(new Set(result.runs.map((run) => run.id))).toEqual( + new Set(["LR-1"]) + ); + + const page = await queryActivityHistory({ + homeDir: home, + rootDir: globalRoot, + scope: "all", + since: "2026-03-01T00:00:00.000Z", + limit: 1, + }); + expect(page.events).toHaveLength(1); + expect(page.runs).toHaveLength(1); + expect(page.runs[0]?.id).toBe("LR-1"); + expect(page.runs[0]?.scopeId).toBe(page.events[0]?.scopeId); }); test("links a superseded correlation branch to its opaque successor", async () => { diff --git a/src/activity-history.ts b/src/activity-history.ts index 84ed7ad..0dda11c 100644 --- a/src/activity-history.ts +++ b/src/activity-history.ts @@ -1747,10 +1747,12 @@ export async function queryActivityHistory( ); const pageEvents = sortedEvents.slice(0, limit); const hasMore = sortedEvents.length > limit; - const returnedRunIds = new Set(pageEvents.map((event) => event.runId)); + const returnedRunKeys = new Set( + pageEvents.map((event) => `${event.scopeId}\0${event.runId}`) + ); const runs = reads .flatMap((read) => read.runs) - .filter((run) => returnedRunIds.has(run.id)) + .filter((run) => returnedRunKeys.has(`${run.scopeId}\0${run.id}`)) .filter( (run, index, values) => values.findIndex( From 5980f438b9cc00f6b9d47b11a85e2a7460e1f9c2 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 16:05:05 -0400 Subject: [PATCH 05/12] docs: clarify scoped history run joins --- docs/activity-history.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/activity-history.md b/docs/activity-history.md index c5a30fb..c6737ec 100644 --- a/docs/activity-history.md +++ b/docs/activity-history.md @@ -38,8 +38,10 @@ field carries the concrete transition, such as `repeated`, `task`, `defer`, Every event references one recorded run. The run record carries scope, trigger, status, loop revision, config revision, reconciliation window, and coverage counts. Resource events carry only the changed field and bounded safe context. -Consumers should join on `runId` and `resource.id`; they should not infer -lineage from titles or internal IDs. +Consumers should join events to run records on the composite `(scopeId, +runId)` key because run IDs are only unique within a scope. Resource lineage +joins use `resource.id`; consumers should not infer lineage from titles or +internal IDs. When correlation merges or another transition supersedes a prior resource, `relatedResourceIds` contains the opaque successor identity. This keeps branch From 5e1a46c3da6fb3874129656d200ac6131a117d2a Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 16:22:37 -0400 Subject: [PATCH 06/12] fix: harden activity history failures --- src/activity-history.test.ts | 26 ++++++++++++++++++++++++++ src/activity-history.ts | 13 ++++++++++--- src/evolution-loop.test.ts | 18 ++++++++++++++++++ src/evolution-loop.ts | 14 ++++++++++++++ 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/activity-history.test.ts b/src/activity-history.test.ts index c21a4fd..2e376ae 100644 --- a/src/activity-history.test.ts +++ b/src/activity-history.test.ts @@ -5,7 +5,9 @@ import { mkdtemp, readdir, readFile, + rename, rm, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -793,6 +795,30 @@ describe("activity history", () => { corruptSegments: 1, }); + await appendRun({ + home, + rootDir: globalRoot, + index: 2, + recordedAt: "2026-07-01T01:00:00.000Z", + items: [], + }); + const segmentNames = await readdir(segmentDir); + const linkedSegmentName = segmentNames.find((name) => name !== segmentName); + const linkedSegmentPath = join(segmentDir, linkedSegmentName!); + const linkedTargetPath = `${linkedSegmentPath}.target`; + await rename(linkedSegmentPath, linkedTargetPath); + await symlink(linkedTargetPath, linkedSegmentPath); + const substituted = await queryActivityHistory({ + homeDir: home, + rootDir: globalRoot, + scope: "global", + since: "2026-07-01T00:00:00.000Z", + }); + expect(substituted.coverage.scopes[0]).toMatchObject({ + state: "degraded", + corruptSegments: 2, + }); + for (let index = 0; index < 51; index += 1) { const projectRoot = join(home, "work", `project-${index}`, ".ai"); await appendRun({ diff --git a/src/activity-history.ts b/src/activity-history.ts index 0dda11c..192fcd7 100644 --- a/src/activity-history.ts +++ b/src/activity-history.ts @@ -952,11 +952,18 @@ async function readBoundedJson( pathValue: string, maxBytes: number ): Promise { + return JSON.parse(await readBoundedText(pathValue, maxBytes)) as unknown; +} + +async function readBoundedText( + pathValue: string, + maxBytes: number +): Promise { const info = await lstat(pathValue); if (!info.isFile() || info.isSymbolicLink() || info.size > maxBytes) { throw new Error("Activity history file is not a bounded regular file"); } - return JSON.parse(await readFile(pathValue, "utf8")) as unknown; + return await readFile(pathValue, "utf8"); } async function fileExists(pathValue: string): Promise { @@ -1139,7 +1146,7 @@ export async function appendActivityHistory(args: { ) { throw error; } - const existing = await readFile(segmentPath, "utf8"); + const existing = await readBoundedText(segmentPath, MAX_SEGMENT_BYTES); if (sha256(existing) !== sha256(body)) { throw new Error( "Activity history run identity conflicts with existing content" @@ -1527,7 +1534,7 @@ async function readScope(args: { } try { const pathValue = join(args.descriptor.segmentDir, record.file); - const raw = await readFile(pathValue, "utf8"); + const raw = await readBoundedText(pathValue, MAX_SEGMENT_BYTES); if ( Buffer.byteLength(raw) > MAX_SEGMENT_BYTES || sha256(raw) !== record.checksum diff --git a/src/evolution-loop.test.ts b/src/evolution-loop.test.ts index 9dbc27e..0e00776 100644 --- a/src/evolution-loop.test.ts +++ b/src/evolution-loop.test.ts @@ -13,6 +13,7 @@ import { tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; import { latestActivityFeed } from "./activity"; import { resolveActivityActionLocator } from "./activity-action"; +import { queryActivityHistory } from "./activity-history"; import { type AiProposalRecord, acceptProposal, @@ -1197,6 +1198,23 @@ describe("evolution loop", () => { expect(Object.keys(state.queue)).toHaveLength(failed.queue.length); expect(state.lastRunStatus).toBe("failed"); expect(state.lastFailure.message).toBe("injected audit failure"); + const history = await queryActivityHistory({ + homeDir: project.homeDir, + rootDir: project.rootDir, + scope: "project", + }); + expect(history.runs).toEqual([ + expect.objectContaining({ + id: failed.runId, + status: "failed", + }), + ]); + expect(history.events).toContainEqual( + expect.objectContaining({ + runId: failed.runId, + action: "run-recorded", + }) + ); }); it("never takes over a live lease and recovers an abandoned lease", async () => { diff --git a/src/evolution-loop.ts b/src/evolution-loop.ts index ff63955..d04c544 100644 --- a/src/evolution-loop.ts +++ b/src/evolution-loop.ts @@ -2186,6 +2186,7 @@ async function runEvolutionLoopScoped(args: { facultAiEvolutionLoopStatePath(args.homeDir, args.rootDir), `${JSON.stringify(nextState, null, 2)}\n` ); + let historyAttempted = false; try { await args.onBeforeAuditCommit?.(); await mkdir(dirname(auditPath), { recursive: true }); @@ -2206,6 +2207,7 @@ async function runEvolutionLoopScoped(args: { })}\n`, "utf8" ); + historyAttempted = true; const { appendActivityHistory } = await import("./activity-history"); await appendActivityHistory({ homeDir: args.homeDir, @@ -2267,6 +2269,18 @@ async function runEvolutionLoopScoped(args: { 2 )}\n` ); + if (!historyAttempted) { + const { appendActivityHistory } = await import( + "./activity-history" + ); + await appendActivityHistory({ + homeDir: args.homeDir, + rootDir: args.rootDir, + report: failedReport, + review, + configRevision: config.generation, + }); + } return failedReport; } } From 79d746d93f343eeb1622885326da780236eb591b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 16:30:29 -0400 Subject: [PATCH 07/12] fix: preserve failed loop reports --- src/evolution-loop.test.ts | 36 ++++++++++++++++++++++++++++++++++++ src/evolution-loop.ts | 24 ++++++++++++++++++------ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/evolution-loop.test.ts b/src/evolution-loop.test.ts index 0e00776..4bea8a8 100644 --- a/src/evolution-loop.test.ts +++ b/src/evolution-loop.test.ts @@ -36,6 +36,7 @@ import { runEvolutionLoop, } from "./evolution-loop"; import { + facultAiActivityHistoryManifestPath, facultAiEvolutionLoopAuditPath, facultAiEvolutionLoopConfigPath, facultAiEvolutionLoopStatePath, @@ -1130,6 +1131,41 @@ describe("evolution loop", () => { }); }); + it("preserves a failed run when activity history is malformed", async () => { + const project = await makeProject(); + await enableEvolutionLoop({ + ...project, + sourceIds: ["missing-source"], + now: () => new Date("2026-01-03T00:00:00.000Z"), + }); + const manifestPath = facultAiActivityHistoryManifestPath( + project.homeDir, + project.rootDir + ); + await mkdir(dirname(manifestPath), { recursive: true }); + await Bun.write(manifestPath, "{malformed\n"); + + const failed = await runEvolutionLoop({ + ...project, + since: "2026-01-01", + until: "2026-01-03", + now: () => new Date("2026-01-03T00:00:00.000Z"), + }); + + expect(failed.status).toBe("failed"); + expect(failed.attempts).toHaveLength(3); + expect(failed.attempts[0]?.error).toContain("missing-source"); + const history = await queryActivityHistory({ + homeDir: project.homeDir, + rootDir: project.rootDir, + scope: "project", + }); + expect(history.coverage.scopes[0]).toMatchObject({ + state: "degraded", + detail: "history-manifest-invalid", + }); + }); + it("records committed mutations when a later materialization step fails", async () => { const project = await makeProject(); await mkdir(join(project.rootDir, "instructions"), { recursive: true }); diff --git a/src/evolution-loop.ts b/src/evolution-loop.ts index d04c544..8f49559 100644 --- a/src/evolution-loop.ts +++ b/src/evolution-loop.ts @@ -1825,6 +1825,22 @@ export async function latestEvolutionLoopReport( ); } +async function appendFailedRunHistory(args: { + homeDir: string; + rootDir: string; + report: EvolutionLoopReport; + review?: ReconciliationReview | null; + configRevision: number; +}): Promise { + try { + const { appendActivityHistory } = await import("./activity-history"); + await appendActivityHistory(args); + } catch { + // A secondary history-store failure must not hide the failed run's root cause. + return; + } +} + async function persistFailedLoopRun(args: { homeDir: string; rootDir: string; @@ -1930,8 +1946,7 @@ async function persistFailedLoopRun(args: { recovery: "Resolve the reported error, inspect the failed report, and rerun the same bounded window", }); - const { appendActivityHistory } = await import("./activity-history"); - await appendActivityHistory({ + await appendFailedRunHistory({ homeDir: args.homeDir, rootDir: args.rootDir, report, @@ -2270,10 +2285,7 @@ async function runEvolutionLoopScoped(args: { )}\n` ); if (!historyAttempted) { - const { appendActivityHistory } = await import( - "./activity-history" - ); - await appendActivityHistory({ + await appendFailedRunHistory({ homeDir: args.homeDir, rootDir: args.rootDir, report: failedReport, From 7d2ac8c75dc2096c68730ccc30a730bba8041c70 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 16:48:09 -0400 Subject: [PATCH 08/12] fix: align loop audit after history failure --- src/evolution-loop.test.ts | 44 ++++++++++++++++++++++++++++++++++++++ src/evolution-loop.ts | 22 +++++++++++++++++-- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/evolution-loop.test.ts b/src/evolution-loop.test.ts index 4bea8a8..7e59bb1 100644 --- a/src/evolution-loop.test.ts +++ b/src/evolution-loop.test.ts @@ -47,6 +47,7 @@ import { } from "./paths"; const SIGNAL_FAMILY_ID_RE = /^SF-/; +const COMPLETED_RUN_STATUS_RE = /^(complete|degraded)$/; const temporaryRoots: string[] = []; async function makeProject(): Promise<{ @@ -1253,6 +1254,49 @@ describe("evolution loop", () => { ); }); + it("records a terminal failed audit when committed history persistence fails", async () => { + const project = await makeProject(); + await enableEvolutionLoop({ ...project }); + const manifestPath = facultAiActivityHistoryManifestPath( + project.homeDir, + project.rootDir + ); + await mkdir(dirname(manifestPath), { recursive: true }); + await Bun.write(manifestPath, "{malformed\n"); + + const failed = await runEvolutionLoop({ + ...project, + since: "2026-01-01", + until: "2026-01-03", + now: () => new Date("2026-01-04T00:00:00.000Z"), + }); + + expect(failed.status).toBe("failed"); + expect(failed.attempts.at(-1)?.error).toContain("post-commit history"); + const audit = ( + await readFile( + facultAiEvolutionLoopAuditPath(project.homeDir, project.rootDir), + "utf8" + ) + ) + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .filter((entry) => entry.runId === failed.runId); + expect(audit.at(-1)).toMatchObject({ + runId: failed.runId, + status: "failed", + }); + expect(audit.at(-2)?.status).toMatch(COMPLETED_RUN_STATUS_RE); + const state = JSON.parse( + await readFile( + facultAiEvolutionLoopStatePath(project.homeDir, project.rootDir), + "utf8" + ) + ); + expect(state.lastRunStatus).toBe("failed"); + }); + it("never takes over a live lease and recovers an abandoned lease", async () => { const project = await makeProject(); await enableEvolutionLoop({ diff --git a/src/evolution-loop.ts b/src/evolution-loop.ts index 8f49559..f928238 100644 --- a/src/evolution-loop.ts +++ b/src/evolution-loop.ts @@ -2239,7 +2239,7 @@ async function runEvolutionLoopScoped(args: { { attempt: attempts.length + 1, ok: false, - error: `post-commit audit: ${message}`, + error: `post-commit ${historyAttempted ? "history" : "audit"}: ${message}`, }, ]; const failedReport: EvolutionLoopReport = { @@ -2284,7 +2284,25 @@ async function runEvolutionLoopScoped(args: { 2 )}\n` ); - if (!historyAttempted) { + if (historyAttempted) { + try { + await appendLoopAudit(args, { + runId, + generatedAt, + trigger: report.trigger, + status: "failed", + generationBefore: report.generationBefore, + generationAfter: report.generationAfter, + attempts: failedAttempts, + mutations: report.mutations, + reportPath, + recovery: + "Repair activity history storage, inspect the failed report, and rerun the same bounded window", + }); + } catch { + // Preserve the authoritative failed report when the audit sink also becomes unavailable. + } + } else { await appendFailedRunHistory({ homeDir: args.homeDir, rootDir: args.rootDir, From d2da380f60112a7475091e7b7c0c847f92599c2e Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 16:58:28 -0400 Subject: [PATCH 09/12] fix: scope aggregate history reads globally --- src/ai-cli.test.ts | 17 +++++++++++++++++ src/ai.ts | 10 ++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/ai-cli.test.ts b/src/ai-cli.test.ts index 2e2eb09..c3110bc 100644 --- a/src/ai-cli.test.ts +++ b/src/ai-cli.test.ts @@ -305,6 +305,23 @@ describe("ai CLI", () => { version: 2, feeds: [{ scopeId: "global", feed: { scope: "global" } }], }); + const allHistoryOut = await captureConsole(async () => { + await aiCommand([ + "loop", + "history", + "--all", + "--root", + rootDir, + "--json", + ]); + }); + expect(allHistoryOut.errors).toEqual([]); + expect(JSON.parse(allHistoryOut.logs.join("\n"))).toMatchObject({ + kind: "activity-history", + filters: { scope: "all" }, + coverage: { configuredScopes: 1 }, + runs: [{ scopeId: "global" }], + }); const implicitAllActivityOut = await captureConsole(async () => { await aiCommand(["loop", "activity", "--root", rootDir, "--json"]); }); diff --git a/src/ai.ts b/src/ai.ts index 793bf33..595f29b 100644 --- a/src/ai.ts +++ b/src/ai.ts @@ -3186,7 +3186,7 @@ async function loopCommand(argv: string[]) { `Unknown activity history event type: ${invalidEventType}` ); } - const result = await queryActivityHistory({ + const queryArgs = { homeDir, rootDir: historyRootDir, scope: allScopes ? "all" : loopScope, @@ -3198,7 +3198,13 @@ async function loopCommand(argv: string[]) { eventTypes as import("./activity-history").ActivityHistoryEventType[], limit: parseIntegerFlag(commandArgs, "--limit"), cursor: parseStringFlag(commandArgs, "--cursor"), - }); + } as const; + const result = allScopes + ? await withFacultRootScope( + { rootDir: historyRootDir, scope: "global" }, + async () => await queryActivityHistory(queryArgs) + ) + : await queryActivityHistory(queryArgs); console.log( json ? JSON.stringify(result, null, 2) : renderActivityHistory(result) ); From d876774efa2d357e6d221277f39423a354f2d485 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 17:16:24 -0400 Subject: [PATCH 10/12] fix: keep history pruning best effort --- src/activity-history.test.ts | 51 ++++++++++++++++++++++++++++++++++++ src/activity-history.ts | 5 +++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/activity-history.test.ts b/src/activity-history.test.ts index 2e376ae..54d3322 100644 --- a/src/activity-history.test.ts +++ b/src/activity-history.test.ts @@ -633,6 +633,57 @@ describe("activity history", () => { expect(retained.coverage.complete).toBe(true); }); + test("keeps a committed run successful when pruned segment cleanup fails", async () => { + const home = await tempHome(); + const rootDir = join(home, ".ai"); + await appendRun({ + home, + rootDir, + index: 1, + recordedAt: "2026-05-01T00:00:00.000Z", + items: [], + retention: { maxAgeDays: 365, maxEvents: 100 }, + }); + const manifestPath = facultAiActivityHistoryManifestPath(home, rootDir); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as { + segments: Array<{ file: string }>; + }; + const staleSegmentPath = join( + facultAiActivityHistorySegmentDir(home, rootDir), + manifest.segments[0]!.file + ); + await rm(staleSegmentPath); + await mkdir(staleSegmentPath); + + const appended = await appendRun({ + home, + rootDir, + index: 2, + recordedAt: "2026-05-02T00:00:00.000Z", + items: [], + retention: { maxAgeDays: 365, maxEvents: 1 }, + }); + + expect(appended.appended).toBe(true); + const retry = await appendRun({ + home, + rootDir, + index: 2, + recordedAt: "2026-05-02T00:00:00.000Z", + items: [], + retention: { maxAgeDays: 365, maxEvents: 1 }, + }); + expect(retry.appended).toBe(false); + const timeline = await queryActivityHistory({ + homeDir: home, + rootDir, + scope: "global", + }); + expect(timeline.runs).toEqual([ + expect.objectContaining({ id: "LR-2", status: "complete" }), + ]); + }); + test("bounds lineage heads and reports item-lineage pruning separately", async () => { const home = await tempHome(); const rootDir = join(home, ".ai"); diff --git a/src/activity-history.ts b/src/activity-history.ts index 192fcd7..f126db8 100644 --- a/src/activity-history.ts +++ b/src/activity-history.ts @@ -1180,7 +1180,10 @@ export async function appendActivityHistory(args: { throw new Error("Activity history manifest exceeds its bounded size"); } await atomicWrite(manifestPath, manifestBody); - await Promise.all( + // The committed manifest is authoritative. Orphaned pruned segments are + // ignored by readers and must not turn a successfully recorded run into a + // failure when best-effort cleanup cannot remove them. + await Promise.allSettled( retained.removed.map(async (entry) => { await rm(join(segmentDir, entry.file), { force: true }); }) From 204bd3a53d246476754716826260e66502f44591 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 17:25:07 -0400 Subject: [PATCH 11/12] fix: bind scoped history reads --- src/ai-cli.test.ts | 17 +++++++++++++++++ src/ai.ts | 13 +++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/ai-cli.test.ts b/src/ai-cli.test.ts index c3110bc..f91862a 100644 --- a/src/ai-cli.test.ts +++ b/src/ai-cli.test.ts @@ -322,6 +322,23 @@ describe("ai CLI", () => { coverage: { configuredScopes: 1 }, runs: [{ scopeId: "global" }], }); + const globalHistoryOut = await captureConsole(async () => { + await aiCommand([ + "loop", + "history", + "--global", + "--root", + rootDir, + "--json", + ]); + }); + expect(globalHistoryOut.errors).toEqual([]); + expect(JSON.parse(globalHistoryOut.logs.join("\n"))).toMatchObject({ + kind: "activity-history", + filters: { scope: "global" }, + coverage: { configuredScopes: 1 }, + runs: [{ scopeId: "global" }], + }); const implicitAllActivityOut = await captureConsole(async () => { await aiCommand(["loop", "activity", "--root", rootDir, "--json"]); }); diff --git a/src/ai.ts b/src/ai.ts index 595f29b..36340d2 100644 --- a/src/ai.ts +++ b/src/ai.ts @@ -3199,12 +3199,13 @@ async function loopCommand(argv: string[]) { limit: parseIntegerFlag(commandArgs, "--limit"), cursor: parseStringFlag(commandArgs, "--cursor"), } as const; - const result = allScopes - ? await withFacultRootScope( - { rootDir: historyRootDir, scope: "global" }, - async () => await queryActivityHistory(queryArgs) - ) - : await queryActivityHistory(queryArgs); + const result = await withFacultRootScope( + { + rootDir: historyRootDir, + scope: allScopes ? "global" : loopScope, + }, + async () => await queryActivityHistory(queryArgs) + ); console.log( json ? JSON.stringify(result, null, 2) : renderActivityHistory(result) ); From c03d64060ee3a9e30400941bd0e51e8ca48c1243 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Wed, 15 Jul 2026 17:33:11 -0400 Subject: [PATCH 12/12] fix: preserve coverage cursor on history failure --- src/evolution-loop.test.ts | 1 + src/evolution-loop.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/evolution-loop.test.ts b/src/evolution-loop.test.ts index 7e59bb1..a5bedfa 100644 --- a/src/evolution-loop.test.ts +++ b/src/evolution-loop.test.ts @@ -1295,6 +1295,7 @@ describe("evolution loop", () => { ) ); expect(state.lastRunStatus).toBe("failed"); + expect(state.lastSuccessfulCoverageUntil).toBeUndefined(); }); it("never takes over a live lease and recovers an abandoned lease", async () => { diff --git a/src/evolution-loop.ts b/src/evolution-loop.ts index f928238..25633c2 100644 --- a/src/evolution-loop.ts +++ b/src/evolution-loop.ts @@ -2274,6 +2274,7 @@ async function runEvolutionLoopScoped(args: { prior.lastSuccessfulScheduledRunAt, lastSuccessfulScheduledConfigGeneration: prior.lastSuccessfulScheduledConfigGeneration, + lastSuccessfulCoverageUntil: prior.lastSuccessfulCoverageUntil, lastFailure: { at: generatedAt, message,