diff --git a/.changeset/hide-stale-cleanup-recovery.md b/.changeset/hide-stale-cleanup-recovery.md new file mode 100644 index 00000000..e5ee0923 --- /dev/null +++ b/.changeset/hide-stale-cleanup-recovery.md @@ -0,0 +1,5 @@ +--- +"@gh-symphony/cli": patch +--- + +Hide stale incomplete-turn recovery from repo status and repo explain after issue workspace cleanup removes the backing workspace for #421. diff --git a/packages/cli/src/commands/repo-explain.ts b/packages/cli/src/commands/repo-explain.ts index d6148582..17143ffc 100644 --- a/packages/cli/src/commands/repo-explain.ts +++ b/packages/cli/src/commands/repo-explain.ts @@ -3,6 +3,7 @@ import { join, resolve } from "node:path"; import type { GlobalOptions } from "../index.js"; import { WorkflowConfigStore, + type IssueWorkspaceRecord, type IssueOrchestrationRecord, type OrchestratorProjectConfig, type OrchestratorRunRecord, @@ -148,13 +149,17 @@ const handler = async ( identifier.trim().toLowerCase() ) ?? null ); - const [issues, issue, issueRecords, runs, snapshot] = await Promise.all([ - issuesPromise, - issuePromise, - readJsonFile(join(runtimeRoot, "issues.json")), - readRuns(runtimeRoot, projectConfig.projectId), - readJsonFile(join(runtimeRoot, "status.json")), - ]); + const [issues, issue, issueRecords, issueWorkspaces, runs, snapshot] = + await Promise.all([ + issuesPromise, + issuePromise, + readJsonFile( + join(runtimeRoot, "issues.json") + ), + readIssueWorkspaces(runtimeRoot, projectConfig.projectId), + readRuns(runtimeRoot, projectConfig.projectId), + readJsonFile(join(runtimeRoot, "status.json")), + ]); const canonicalIssues = resolveCanonicalSubjectIssues(issues); const canonicalIssue = canonicalIssues.find((candidate) => @@ -191,6 +196,7 @@ const handler = async ( allIssues: canonicalIssues, lifecycle: workflow.lifecycle, issueRecords: issueRecords ?? [], + issueWorkspaces, runs, activeRunCount, maxConcurrentAgents: workflow.maxConcurrentAgents, @@ -361,6 +367,30 @@ async function readRuns( ); } +async function readIssueWorkspaces( + runtimeRoot: string, + projectId: string +): Promise { + let workspaceKeys: string[]; + try { + workspaceKeys = await readdir(runtimeRoot); + } catch { + return []; + } + + const records = await Promise.all( + workspaceKeys.map((workspaceKey) => + readJsonFile( + join(runtimeRoot, workspaceKey, "workspace.json") + ) + ) + ); + return records.filter( + (record): record is IssueWorkspaceRecord => + record !== null && record.projectId === projectId + ); +} + async function readJsonFile(path: string): Promise { try { return JSON.parse(await readFile(path, "utf8")) as T; diff --git a/packages/cli/src/commands/status.test.ts b/packages/cli/src/commands/status.test.ts index d96a02b5..24a025b6 100644 --- a/packages/cli/src/commands/status.test.ts +++ b/packages/cli/src/commands/status.test.ts @@ -172,6 +172,56 @@ describe("status command", () => { expect(stdout.output()).toContain("Tokens: 600 / 1,700 total"); }); + it("does not render incomplete-turn recovery when snapshot recovery is null", async () => { + const configDir = await createConfigFixture(); + const projectId = "tenant-a"; + await writeFile( + join(configDir, "status.json"), + JSON.stringify( + { + projectId, + slug: projectId, + tracker: { adapter: "github-project", bindingId: "project-1" }, + lastTickAt: "2026-03-30T11:00:00.000Z", + health: "idle", + summary: { + dispatched: 2, + suppressed: 1, + recovered: 0, + activeRuns: 0, + }, + activeRuns: [], + retryQueue: [], + codexTotals: { + inputTokens: 1400, + outputTokens: 300, + totalTokens: 1700, + secondsRunning: 60, + }, + lastError: null, + recovery: null, + }, + null, + 2 + ) + "\n", + "utf8" + ); + const stdout = captureWrites(process.stdout); + + try { + await statusCommand([], { + configDir, + verbose: false, + json: false, + noColor: true, + }); + } finally { + stdout.restore(); + } + + expect(stdout.output()).not.toContain("Recoverable incomplete turn:"); + }); + it("renders incomplete-turn recovery details in text status output", async () => { const configDir = await createConfigFixture(); const projectId = "tenant-a"; @@ -249,6 +299,58 @@ describe("status command", () => { ); }); + it("does not render incomplete-turn recovery details when snapshot recovery is null", async () => { + const configDir = await createConfigFixture(); + const projectId = "tenant-a"; + await writeFile( + join(configDir, "status.json"), + JSON.stringify( + { + projectId, + slug: projectId, + tracker: { adapter: "github-project", bindingId: "project-1" }, + lastTickAt: "2026-03-30T11:00:00.000Z", + health: "idle", + summary: { + dispatched: 2, + suppressed: 1, + recovered: 0, + activeRuns: 0, + }, + activeRuns: [], + retryQueue: [], + codexTotals: { + inputTokens: 1400, + outputTokens: 300, + totalTokens: 1700, + secondsRunning: 60, + }, + lastError: null, + recovery: null, + }, + null, + 2 + ) + "\n", + "utf8" + ); + const stdout = captureWrites(process.stdout); + + try { + await statusCommand([], { + configDir, + verbose: false, + json: false, + noColor: true, + }); + } finally { + stdout.restore(); + } + + const output = stdout.output(); + expect(output).not.toContain("Recoverable incomplete turn:"); + expect(output).not.toContain("Workspace /tmp/work/repository"); + }); + it("falls back to the legacy per-project status snapshot path", async () => { const configDir = await createConfigFixture("legacy"); const stdout = captureWrites(process.stdout); diff --git a/packages/core/src/observability/snapshot-builder.test.ts b/packages/core/src/observability/snapshot-builder.test.ts index 0db77a31..4eb4d7d5 100644 --- a/packages/core/src/observability/snapshot-builder.test.ts +++ b/packages/core/src/observability/snapshot-builder.test.ts @@ -7,6 +7,7 @@ import type { OrchestratorProjectConfig, OrchestratorRunRecord, } from "../contracts/status-surface.js"; +import type { IssueWorkspaceRecord } from "../domain/issue.js"; /** * Helper to create a minimal OrchestratorProjectConfig for testing @@ -69,6 +70,25 @@ function mockRun( }; } +function mockIssueWorkspace( + overrides?: Partial +): IssueWorkspaceRecord { + return { + workspaceKey: "key-001", + projectId: "tenant-123", + adapter: "github", + issueSubjectId: "subject-001", + issueIdentifier: "acme/platform#42", + workspacePath: "/tmp/workspace", + repositoryPath: "/tmp/work/repository", + status: "active", + createdAt: "2024-01-01T00:00:00Z", + updatedAt: "2024-01-01T00:00:00Z", + lastError: null, + ...overrides, + }; +} + describe("buildProjectSnapshot", () => { it("returns idle health when no active runs and no error", () => { const input: SnapshotInput = { @@ -382,6 +402,90 @@ describe("buildProjectSnapshot", () => { expect(snapshot.recovery).toEqual(latestRun.recovery); }); + it("does not surface incomplete-turn recovery for a removed issue workspace", () => { + const run = mockRun({ + runId: "run-removed", + status: "suppressed", + issueWorkspaceKey: "key-removed", + updatedAt: "2024-01-01T00:07:00Z", + recovery: { + kind: "incomplete-turn-dirty-workspace", + runId: "run-removed", + issueId: "issue-001", + issueIdentifier: "acme/platform#42", + workspacePath: "/tmp/work/removed-repository", + dirtyFiles: ["partial.txt"], + lastEvent: "heartbeat", + lastEventAt: "2024-01-01T00:06:30Z", + sessionId: "session-removed", + threadId: "thread-removed", + suggestedCommand: + "cd /tmp/work/removed-repository && git status --short && git diff", + detectedAt: "2024-01-01T00:07:00Z", + }, + }); + + const snapshot = buildProjectSnapshot({ + project: mockProject(), + activeRuns: [], + allRuns: [run], + issueWorkspaces: [ + mockIssueWorkspace({ + workspaceKey: "key-removed", + repositoryPath: "/tmp/work/removed-repository", + status: "removed", + }), + ], + summary: { dispatched: 0, suppressed: 1, recovered: 0 }, + lastTickAt: "2024-01-01T00:10:00Z", + lastError: null, + }); + + expect(snapshot.recovery).toBeNull(); + }); + + it("surfaces incomplete-turn recovery for an active issue workspace", () => { + const run = mockRun({ + runId: "run-active", + status: "suppressed", + issueWorkspaceKey: "key-active", + updatedAt: "2024-01-01T00:07:00Z", + recovery: { + kind: "incomplete-turn-dirty-workspace", + runId: "run-active", + issueId: "issue-001", + issueIdentifier: "acme/platform#42", + workspacePath: "/tmp/work/active-repository", + dirtyFiles: ["partial.txt"], + lastEvent: "heartbeat", + lastEventAt: "2024-01-01T00:06:30Z", + sessionId: "session-active", + threadId: "thread-active", + suggestedCommand: + "cd /tmp/work/active-repository && git status --short && git diff", + detectedAt: "2024-01-01T00:07:00Z", + }, + }); + + const snapshot = buildProjectSnapshot({ + project: mockProject(), + activeRuns: [], + allRuns: [run], + issueWorkspaces: [ + mockIssueWorkspace({ + workspaceKey: "key-active", + repositoryPath: "/tmp/work/active-repository", + status: "active", + }), + ], + summary: { dispatched: 0, suppressed: 1, recovered: 0 }, + lastTickAt: "2024-01-01T00:10:00Z", + lastError: null, + }); + + expect(snapshot.recovery).toEqual(run.recovery); + }); + it("does not surface stale recovery after the recovery run completes", () => { const suppressedRun = mockRun({ runId: "run-suppressed", diff --git a/packages/core/src/observability/snapshot-builder.ts b/packages/core/src/observability/snapshot-builder.ts index 33da6e78..cd33334c 100644 --- a/packages/core/src/observability/snapshot-builder.ts +++ b/packages/core/src/observability/snapshot-builder.ts @@ -7,10 +7,12 @@ */ import type { + IncompleteTurnRecoveryInfo, OrchestratorRunRecord, OrchestratorProjectConfig, ProjectStatusSnapshot, } from "../contracts/status-surface.js"; +import type { IssueWorkspaceRecord } from "../domain/issue.js"; export type SnapshotInput = { project: OrchestratorProjectConfig; @@ -24,6 +26,7 @@ export type SnapshotInput = { lastTickAt: string; lastError: string | null; rateLimits?: Record | null; + issueWorkspaces?: readonly IssueWorkspaceRecord[]; }; /** @@ -43,6 +46,7 @@ export function buildProjectSnapshot( lastTickAt, lastError, rateLimits, + issueWorkspaces, } = input; const cumulativeTokenUsageByIssue = aggregateTokenUsageByIssue( allRuns ?? activeRuns @@ -92,7 +96,10 @@ export function buildProjectSnapshot( retryKind: run.retryKind ?? "failure", nextRetryAt: run.nextRetryAt, })), - recovery: findLatestRecovery([...(allRuns ?? []), ...activeRuns]), + recovery: findLatestRecovery( + [...(allRuns ?? []), ...activeRuns], + issueWorkspaces ?? [] + ), lastError, codexTotals: aggregateTokenUsage(allRuns ?? activeRuns, lastTickAt), rateLimits: rateLimits ?? null, @@ -100,11 +107,12 @@ export function buildProjectSnapshot( } function findLatestRecovery( - runs: OrchestratorRunRecord[] + runs: OrchestratorRunRecord[], + issueWorkspaces: readonly IssueWorkspaceRecord[] ): ProjectStatusSnapshot["recovery"] { return ( [...runs] - .filter((run) => isUnresolvedRecoveryRun(run, runs)) + .filter((run) => isUnresolvedRecoveryRun(run, runs, issueWorkspaces)) .sort((left, right) => { const leftTime = new Date(left.updatedAt).getTime(); const rightTime = new Date(right.updatedAt).getTime(); @@ -116,12 +124,17 @@ function findLatestRecovery( function isUnresolvedRecoveryRun( run: OrchestratorRunRecord, - runs: OrchestratorRunRecord[] + runs: OrchestratorRunRecord[], + issueWorkspaces: readonly IssueWorkspaceRecord[] ): boolean { if (!run.recovery) { return false; } + if (!isRecoveryWorkspaceActionable(run, run.recovery, issueWorkspaces)) { + return false; + } + if ( run.status === "suppressed" && runs.some( @@ -145,6 +158,61 @@ function isUnresolvedRecoveryRun( ); } +export function isRecoveryWorkspaceActionable( + run: OrchestratorRunRecord, + recovery: IncompleteTurnRecoveryInfo, + issueWorkspaces: readonly IssueWorkspaceRecord[] +): boolean { + const workspaceRecord = findRecoveryWorkspaceRecord( + run, + recovery, + issueWorkspaces + ); + + return !workspaceRecord || workspaceRecord.status === "active"; +} + +function findRecoveryWorkspaceRecord( + run: OrchestratorRunRecord, + recovery: IncompleteTurnRecoveryInfo, + issueWorkspaces: readonly IssueWorkspaceRecord[] +): IssueWorkspaceRecord | null { + const projectWorkspaces = issueWorkspaces.filter( + (workspace) => workspace.projectId === run.projectId + ); + + if (run.issueWorkspaceKey) { + const byKey = projectWorkspaces.find( + (workspace) => workspace.workspaceKey === run.issueWorkspaceKey + ); + if (byKey) { + return byKey; + } + } + + const byPath = projectWorkspaces.find( + (workspace) => + workspace.repositoryPath === recovery.workspacePath || + workspace.workspacePath === recovery.workspacePath + ); + if (byPath) { + return byPath; + } + + const bySubject = projectWorkspaces.find( + (workspace) => workspace.issueSubjectId === run.issueSubjectId + ); + if (bySubject) { + return bySubject; + } + + return ( + projectWorkspaces.find( + (workspace) => workspace.issueIdentifier === recovery.issueIdentifier + ) ?? null + ); +} + function aggregateTokenUsageByIssue( runs: OrchestratorRunRecord[] ): Map> { diff --git a/packages/orchestrator/src/dispatch.test.ts b/packages/orchestrator/src/dispatch.test.ts index f9226967..af22526c 100644 --- a/packages/orchestrator/src/dispatch.test.ts +++ b/packages/orchestrator/src/dispatch.test.ts @@ -11,6 +11,7 @@ import type { OrchestratorProjectConfig, TrackedIssue, TrackedPullRequestContext, + IssueWorkspaceRecord, } from "@gh-symphony/core"; import { OrchestratorFsStore } from "./fs-store.js"; import * as trackerAdapters from "./tracker-adapters.js"; @@ -83,6 +84,25 @@ function makeRun( }; } +function makeIssueWorkspace( + overrides: Partial & { workspaceKey: string } +): IssueWorkspaceRecord { + return { + workspaceKey: overrides.workspaceKey, + projectId: "tenant-1", + adapter: "github-project", + issueSubjectId: "issue-1", + issueIdentifier: "acme/repo#1", + workspacePath: "/tmp/workspace", + repositoryPath: "/tmp/work", + status: "active", + createdAt: "2026-03-09T00:00:00.000Z", + updatedAt: "2026-03-09T00:00:00.000Z", + lastError: null, + ...overrides, + }; +} + function makePullRequestContext( repository: { owner: string; name: string; cloneUrl: string; path: string }, number: number, @@ -368,6 +388,60 @@ describe("explainIssueDispatch", () => { ); }); + it("does not warn about dirty-workspace recovery for a removed issue workspace", () => { + const issue = makeIssue({ id: "issue-1", identifier: "acme/repo#1" }); + const run = makeRun({ + runId: "run-incomplete", + issueId: issue.id, + status: "suppressed", + updatedAt: "2026-03-09T00:05:00.000Z", + completedAt: "2026-03-09T00:05:00.000Z", + issueWorkspaceKey: "workspace-1", + recovery: { + kind: "incomplete-turn-dirty-workspace", + runId: "run-incomplete", + issueId: issue.id, + issueIdentifier: issue.identifier, + workspacePath: "/tmp/work", + dirtyFiles: ["partial.txt"], + lastEvent: "heartbeat", + lastEventAt: "2026-03-09T00:04:30.000Z", + sessionId: "session-1", + threadId: "thread-1", + suggestedCommand: "cd /tmp/work && git status --short && git diff", + detectedAt: "2026-03-09T00:05:00.000Z", + }, + }); + const report = explainIssueDispatch({ + identifier: issue.identifier, + issue, + projectRepository, + allIssues: [issue], + lifecycle, + issueRecords: [], + issueWorkspaces: [ + makeIssueWorkspace({ + workspaceKey: "workspace-1", + repositoryPath: "/tmp/work", + status: "removed", + }), + ], + runs: [run], + activeRunCount: 0, + maxConcurrentAgents: 3, + maxConcurrentAgentsByState: {}, + }); + + expect(report.checks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "runtime_ownership", status: "pass" }), + ]) + ); + expect(JSON.stringify(report)).not.toContain( + "dispatch will start a recovery turn" + ); + }); + it("explains project concurrency limits", () => { const issue = makeIssue({ id: "issue-1", identifier: "acme/repo#1" }); const report = explainIssueDispatch({ diff --git a/packages/orchestrator/src/explain.ts b/packages/orchestrator/src/explain.ts index 053dd447..20d66486 100644 --- a/packages/orchestrator/src/explain.ts +++ b/packages/orchestrator/src/explain.ts @@ -1,7 +1,9 @@ import { isStateActive, isStateTerminal, + isRecoveryWorkspaceActionable, matchesWorkflowState, + type IssueWorkspaceRecord, type IssueOrchestrationRecord, type OrchestratorRunRecord, type RepositoryRef, @@ -52,6 +54,7 @@ export type ExplainDispatchInput = { allIssues: readonly TrackedIssue[]; lifecycle: WorkflowLifecycleConfig; issueRecords: readonly IssueOrchestrationRecord[]; + issueWorkspaces?: readonly IssueWorkspaceRecord[]; runs: readonly OrchestratorRunRecord[]; activeRunCount: number; maxConcurrentAgents: number; @@ -99,7 +102,14 @@ export function explainIssueDispatch( checks.push(explainWorkflowState(issue, input.lifecycle)); checks.push(explainBlockers(issue, input.lifecycle, input.allIssues)); - checks.push(explainRuntimeOwnership(issue, input.issueRecords, input.runs)); + checks.push( + explainRuntimeOwnership( + issue, + input.issueRecords, + input.runs, + input.issueWorkspaces ?? [] + ) + ); checks.push( explainDispatchLimits( issue, @@ -345,7 +355,8 @@ function explainBlockers( function explainRuntimeOwnership( issue: TrackedIssue, issueRecords: readonly IssueOrchestrationRecord[], - runs: readonly OrchestratorRunRecord[] + runs: readonly OrchestratorRunRecord[], + issueWorkspaces: readonly IssueWorkspaceRecord[] ): DispatchExplainCheck { const record = issueRecords.find( (candidate) => @@ -439,7 +450,12 @@ function explainRuntimeOwnership( if ( latestRun?.status === "suppressed" && - latestRun.recovery?.kind === "incomplete-turn-dirty-workspace" + latestRun.recovery?.kind === "incomplete-turn-dirty-workspace" && + isRecoveryWorkspaceActionable( + latestRun, + latestRun.recovery, + issueWorkspaces + ) ) { return { id: "runtime_ownership", diff --git a/packages/orchestrator/src/service.test.ts b/packages/orchestrator/src/service.test.ts index b3d96452..7c64c565 100644 --- a/packages/orchestrator/src/service.test.ts +++ b/packages/orchestrator/src/service.test.ts @@ -1317,6 +1317,46 @@ describe("OrchestratorService", () => { updatedAt: "2026-03-08T00:00:00.000Z", lastError: null, }); + await store.saveRun({ + runId: "run-incomplete", + projectId: "tenant-1", + projectSlug: "tenant-1", + issueId: "issue-1", + issueSubjectId: "issue-1", + issueIdentifier: "acme/platform#1", + issueState: "In progress", + repository, + status: "suppressed", + attempt: 1, + processId: null, + port: null, + workingDirectory: repositoryPath, + issueWorkspaceKey: workspaceKey, + workspaceRuntimeDir: join(tempRoot, "run-incomplete", "workspace"), + workflowPath: null, + retryKind: null, + createdAt: "2026-03-08T00:00:00.000Z", + updatedAt: "2026-03-08T00:01:00.000Z", + startedAt: "2026-03-08T00:00:00.000Z", + completedAt: "2026-03-08T00:01:00.000Z", + lastError: + "Run suppressed with recoverable incomplete-turn dirty workspace.", + nextRetryAt: null, + recovery: { + kind: "incomplete-turn-dirty-workspace", + runId: "run-incomplete", + issueId: "issue-1", + issueIdentifier: "acme/platform#1", + workspacePath: repositoryPath, + dirtyFiles: ["sentinel.txt"], + lastEvent: "heartbeat", + lastEventAt: "2026-03-08T00:00:30.000Z", + sessionId: "session-1", + threadId: "thread-1", + suggestedCommand: `cd '${repositoryPath}' && git status --short && git diff`, + detectedAt: "2026-03-08T00:01:00.000Z", + }, + }); const service = new OrchestratorService(store, projectConfig, { fetchImpl: vi @@ -1338,8 +1378,15 @@ describe("OrchestratorService", () => { "tenant-1", workspaceKey ); + const preservedRun = await store.loadRun("run-incomplete", "tenant-1"); + const savedStatus = await store.loadProjectStatus(); await expect(readFile(sentinelPath, "utf8")).rejects.toThrow(); expect(workspaceRecord?.status).toBe("removed"); + expect(savedStatus?.recovery).toBeNull(); + expect(preservedRun?.recovery).toMatchObject({ + kind: "incomplete-turn-dirty-workspace", + workspacePath: repositoryPath, + }); }); it("logs and ignores before_remove hook failures during startup cleanup", async () => { diff --git a/packages/orchestrator/src/service.ts b/packages/orchestrator/src/service.ts index de21e1ac..ea4c523a 100644 --- a/packages/orchestrator/src/service.ts +++ b/packages/orchestrator/src/service.ts @@ -999,6 +999,9 @@ export class OrchestratorService { const allTenantRuns = (await this.store.loadAllRuns()).filter( (run) => run.projectId === tenant.projectId ); + const issueWorkspaces = await this.store.loadIssueWorkspaces( + tenant.projectId + ); const latestRuns = allTenantRuns.filter((run) => isActiveRunRecordStatus(run.status) ); @@ -1011,6 +1014,7 @@ export class OrchestratorService { lastTickAt: now.toISOString(), lastError, rateLimits, + issueWorkspaces, }); await this.store.saveProjectStatus(status); return status;