fix(snapshot-tests): Guard CLI snapshot process output (#421)

Fail CLI domain snapshot invocations when the CLI process itself fails,
is terminated, has no exit status, or emits unexpected stderr. This keeps
fixtures focused on domain tool output while preventing hidden process-level
noise from being ignored.

Co-authored-by: OpenAI Codex <codex@openai.com>

Author: cameroncooke

src/snapshot-tests/__tests__/json-harness-error-state.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { resolveCliJsonSnapshotErrorState } from '../harness.ts';
+import { assertCliSnapshotProcessResult, resolveCliJsonSnapshotErrorState } from '../harness.ts';
 import { resolveMcpSnapshotErrorState } from '../mcp-harness.ts';
 import type { StructuredOutputEnvelope } from '../../types/structured-output.ts';
 
@@ -17,6 +17,42 @@ const errorEnvelope: StructuredOutputEnvelope<null> = {
   error: 'Failed',
 };
 
+describe('CLI snapshot process result guard', () => {
+  it('accepts completed domain invocations without process stderr', () => {
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: undefined, signal: null, status: 1, stderr: '' },
+        'tool',
+      ),
+    ).not.toThrow();
+  });
+
+  it('rejects process stderr so domain snapshots cannot hide user-visible noise', () => {
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: undefined, signal: null, status: 0, stderr: 'warning\n' },
+        'tool',
+      ),
+    ).toThrow('CLI process emitted unexpected stderr for tool:\nwarning');
+  });
+
+  it('rejects failed process execution before snapshot matching', () => {
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: new Error('spawn failed'), signal: null, status: null, stderr: '' },
+        'tool',
+      ),
+    ).toThrow('CLI process failed for tool: spawn failed');
+
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: undefined, signal: 'SIGTERM', status: null, stderr: '' },
+        'tool',
+      ),
+    ).toThrow('CLI process for tool was terminated by signal SIGTERM.');
+  });
+});
+
 describe('JSON snapshot harness error state', () => {
   it('uses CLI process status and envelope.didError when they agree', () => {
     expect(resolveCliJsonSnapshotErrorState(0, successEnvelope, 'tool')).toBe(false);

src/snapshot-tests/harness.ts

@@ -56,6 +56,34 @@ function runSnapshotCli(
   });
 }
 
+function readProcessOutput(output: string | Buffer | null | undefined): string {
+  return typeof output === 'string' ? output : (output?.toString('utf8') ?? '');
+}
+
+export function assertCliSnapshotProcessResult(
+  result: Pick<ReturnType<typeof spawnSync>, 'error' | 'signal' | 'status' | 'stderr'>,
+  label: string,
+): void {
+  if (result.error) {
+    throw new Error(`CLI process failed for ${label}: ${result.error.message}`);
+  }
+
+  if (result.signal) {
+    throw new Error(`CLI process for ${label} was terminated by signal ${result.signal}.`);
+  }
+
+  if (result.status === null) {
+    throw new Error(
+      `CLI process exit status was null for ${label}; the process may have timed out or been killed by a signal.`,
+    );
+  }
+
+  const stderr = readProcessOutput(result.stderr).trim();
+  if (stderr.length > 0) {
+    throw new Error(`CLI process emitted unexpected stderr for ${label}:\n${stderr}`);
+  }
+}
+
 function parseStructuredEnvelope(
   stdout: string,
   label: string,
@@ -107,9 +135,10 @@ export async function createSnapshotHarness(
       throw new Error(`Tool '${cliToolName}' in workflow '${workflow}' is not CLI-available`);
     }
 
+    const label = `${workflow}/${cliToolName}`;
     const result = runSnapshotCli(workflow, cliToolName, args, 'text', options);
-    const stdout =
-      typeof result.stdout === 'string' ? result.stdout : (result.stdout?.toString('utf8') ?? '');
+    assertCliSnapshotProcessResult(result, label);
+    const stdout = readProcessOutput(result.stdout);
 
     return {
       text: normalizeSnapshotOutput(stdout),
@@ -141,19 +170,16 @@ export async function createCliJsonSnapshotHarness(
       throw new Error(`Tool '${cliToolName}' in workflow '${workflow}' is not CLI-available`);
     }
 
+    const label = `${workflow}/${cliToolName}`;
     const result = runSnapshotCli(workflow, cliToolName, args, 'json', options);
-    const stdout =
-      typeof result.stdout === 'string' ? result.stdout : (result.stdout?.toString('utf8') ?? '');
-    const envelope = parseStructuredEnvelope(stdout, `${workflow}/${cliToolName}`);
+    assertCliSnapshotProcessResult(result, label);
+    const stdout = readProcessOutput(result.stdout);
+    const envelope = parseStructuredEnvelope(stdout, label);
 
     return {
       text: formatStructuredEnvelopeFixture(envelope),
       rawText: stdout,
-      isError: resolveCliJsonSnapshotErrorState(
-        result.status,
-        envelope,
-        `${workflow}/${cliToolName}`,
-      ),
+      isError: resolveCliJsonSnapshotErrorState(result.status, envelope, label),
       structuredEnvelope: envelope,
     };
   }