From f9a47a1b31810fd75dce67c5f058d72e16e5e1f0 Mon Sep 17 00:00:00 2001 From: camjac251 Date: Sat, 15 Aug 2026 01:50:51 -0400 Subject: [PATCH] fix(patches): verify reused native artifacts Reused builds are checked in a fresh process where mutation counters do not exist. Separate mutation evidence from serialized artifact checks while retaining exact structural and cardinality guards. --- src/patch-runner.ts | 2 +- src/patches/claudemd-strong.test.ts | 34 +++++ src/patches/claudemd-strong.ts | 9 +- src/patches/no-collapse.test.ts | 130 ++++++++++++++++++-- src/patches/no-collapse.ts | 62 ++++++---- src/types.ts | 14 ++- src/verification/verify-cli-anchors.test.ts | 28 +++++ src/verification/verify-cli-anchors.ts | 2 +- 8 files changed, 243 insertions(+), 38 deletions(-) diff --git a/src/patch-runner.ts b/src/patch-runner.ts index 1489d98..3ddfe35 100644 --- a/src/patch-runner.ts +++ b/src/patch-runner.ts @@ -303,7 +303,7 @@ export class PatchRunner { const verifyStart = performance.now(); const verificationOutput = patch.verifyWithWitness ? patch.verifyWithWitness(output, ast) - : patch.verify(output, ast); + : patch.verify(output, ast, { phase: "mutation" }); verifyTimings.set(patch.tag, performance.now() - verifyStart); const meta = getPatchMetadata(patch.tag); const outcome = normalizeVerificationOutcome(verificationOutput); diff --git a/src/patches/claudemd-strong.test.ts b/src/patches/claudemd-strong.test.ts index cdfe9b9..06d14a4 100644 --- a/src/patches/claudemd-strong.test.ts +++ b/src/patches/claudemd-strong.test.ts @@ -83,6 +83,40 @@ test("claudemd-strong disables subagent CLAUDE.md omission", async () => { assert.equal(claudeMdSystemPrompt.verify(verifiedCode, ast), true); }); +test("claudemd-strong verifies an already-patched artifact without run-local mutation state", async () => { + const ast = parse(SUBAGENT_OMIT_FIXTURE); + await runClaudeMdStrongViaPasses(ast); + const output = print(ast); + const verifiedCode = `${STRONG_DISCLAIMER_LINES.join("\n")}\n${output}`; + + // Reset the run-local counter without mutating the serialized artifact. + await claudeMdSystemPrompt.astPasses?.(parse("const untouched = true;")); + const patchRunResult = claudeMdSystemPrompt.verify(verifiedCode, ast); + assert.equal( + String(patchRunResult).includes( + "No subagent CLAUDE.md omission gate was found to neutralize", + ), + true, + ); + assert.equal( + claudeMdSystemPrompt.verify(verifiedCode, ast, { phase: "artifact" }), + true, + ); + + const unpatchedAst = parse(SUBAGENT_OMIT_FIXTURE); + const unpatchedArtifactResult = claudeMdSystemPrompt.verify( + `${STRONG_DISCLAIMER_LINES.join("\n")}\n${SUBAGENT_OMIT_FIXTURE}`, + unpatchedAst, + { phase: "artifact" }, + ); + assert.equal( + String(unpatchedArtifactResult).includes( + "Subagent CLAUDE.md omission gate is still present", + ), + true, + ); +}); + test("claudemd-strong verify rejects a surviving subagent CLAUDE.md omission gate", () => { const ast = parse(SUBAGENT_OMIT_FIXTURE); const result = claudeMdSystemPrompt.verify( diff --git a/src/patches/claudemd-strong.ts b/src/patches/claudemd-strong.ts index b6f8f96..1e14940 100644 --- a/src/patches/claudemd-strong.ts +++ b/src/patches/claudemd-strong.ts @@ -121,7 +121,7 @@ export const claudeMdSystemPrompt: Patch = { ]; }, - verify: (code, ast) => { + verify: (code, ast, context) => { if (code.includes(WEAK_DISCLAIMER)) { return "Weak CLAUDE.md disclaimer still present (replacement failed)"; } @@ -138,9 +138,10 @@ export const claudeMdSystemPrompt: Patch = { if (survivingGates > 0) { return `Subagent CLAUDE.md omission gate is still present (${survivingGates} surviving)`; } - // A run that neutralizes nothing means the gate moved or the matcher - // went stale; either way the omission behavior would ship live. - if (gatesNeutralized === 0) { + // Mutation verification must prove that the matcher ran. Serialized + // artifacts cannot carry this process-local counter, so their safety + // is established by the surviving-gate check above. + if (context?.phase !== "artifact" && gatesNeutralized === 0) { return "No subagent CLAUDE.md omission gate was found to neutralize"; } } diff --git a/src/patches/no-collapse.test.ts b/src/patches/no-collapse.test.ts index 4bac6d9..4366cfe 100644 --- a/src/patches/no-collapse.test.ts +++ b/src/patches/no-collapse.test.ts @@ -78,34 +78,148 @@ test("no-collapse patches guard while preserving classification isCollapsible", // Memory write flags flipped to false assert.equal(output.includes("isCollapsible: !1"), true); - assert.equal(output.includes("isMemoryWrite: !1"), true); + assert.equal(output.includes("isMemoryWrite: !!0"), true); // Verify passes on patched output assert.equal(noCollapse.verify(output, ast), true); }); +test("no-collapse verifies an already-patched artifact without run-local mutation state", async () => { + const ast = parse(NO_COLLAPSE_FIXTURE); + await runNoCollapseViaPasses(ast); + const output = print(ast); + const artifactAst = parse(output); + + // Reset the run-local counter without mutating the serialized artifact. + await noCollapse.astPasses?.(parse("const untouched = true;")); + const patchRunResult = noCollapse.verify(output, artifactAst); + assert.equal( + patchRunResult, + "Expected exactly one memory write mutation this run, found 0", + ); + assert.equal( + noCollapse.verify(output, artifactAst, { phase: "artifact" }), + true, + ); + + const regressed = output.replace( + "A.isREPL || A.isMemoryWrite", + "A.isCollapsible || A.isREPL", + ); + assert.notEqual(regressed, output); + const artifactRegressionResult = noCollapse.verify( + regressed, + parse(regressed), + { phase: "artifact" }, + ); + assert.equal( + String(artifactRegressionResult).includes( + "Original collapse-metadata guard", + ), + true, + ); + + for (const [field, regressedArtifact] of [ + ["isCollapsible", output.replace("isCollapsible: !1", "isCollapsible: !0")], + [ + "isMemoryWrite", + output.replace("isMemoryWrite: !!0", "isMemoryWrite: !0"), + ], + ] as const) { + assert.notEqual(regressedArtifact, output); + const halfRegressionResult = noCollapse.verify( + regressedArtifact, + parse(regressedArtifact), + { phase: "artifact" }, + ); + assert.equal( + String(halfRegressionResult) + .toLowerCase() + .includes("memory write result"), + true, + `Expected ${field} half-regression failure, got: ${halfRegressionResult}`, + ); + } +}); + test("no-collapse collects verification markers in one inventory", async () => { const ast = parse(NO_COLLAPSE_FIXTURE); await runNoCollapseViaPasses(ast); assert.deepEqual(collectNoCollapseVerification(ast), { - foundMemoryWriteResult: true, - foundUnpatchedMemoryWriteResult: false, + patchedMemoryWriteResultCount: 1, + unpatchedMemoryWriteResultCount: 0, foundOriginalGuard: false, foundPatchedGuard: true, foundClassificationTail: true, }); }); +test("no-collapse rejects duplicate memory-write targets and a partial artifact regression", async () => { + const duplicateTargetFixture = `${NO_COLLAPSE_FIXTURE} +function renderSecondMemoryWriteResult(H, A) { + if (H.type !== "memory_write_duplicate") return null; + return { + filePath: A, + isCollapsible: !0, + isMemoryWrite: !0, + isSearch: !1, + isRead: !1, + isREPL: !1, + }; +} +`; + const ast = parse(duplicateTargetFixture); + await runNoCollapseViaPasses(ast); + const output = print(ast); + assert.equal(output.split("isMemoryWrite: !!0").length - 1, 2); + + const mutationResult = noCollapse.verify(output, ast); + assert.equal( + String(mutationResult).includes( + "Expected exactly one patched memory write result marker", + ), + true, + ); + + await noCollapse.astPasses?.(parse("const untouched = true;")); + const artifactResult = noCollapse.verify(output, parse(output), { + phase: "artifact", + }); + assert.equal( + String(artifactResult).includes( + "Expected exactly one patched memory write result marker", + ), + true, + ); + + const partiallyRegressed = output.replace( + "isMemoryWrite: !!0", + "isMemoryWrite: !0", + ); + assert.notEqual(partiallyRegressed, output); + const partialRegressionResult = noCollapse.verify( + partiallyRegressed, + parse(partiallyRegressed), + { phase: "artifact" }, + ); + assert.equal( + String(partialRegressionResult).includes( + "Memory write result object is not fully patched", + ), + true, + ); +}); + test("no-collapse verify rejects unpatched fixture", () => { const ast = parse(NO_COLLAPSE_FIXTURE); const result = noCollapse.verify(NO_COLLAPSE_FIXTURE, ast); assert.equal(typeof result, "string"); - // Should detect the original guard or unpatched memory write flags. + // Should detect the original guard or unpatched memory write result. assert.equal( typeof result === "string" && (result.includes("Original collapse-metadata guard") || - result.includes("Unpatched memory write result object")), + result.includes("Memory write result object is not fully patched")), true, `Expected unpatched pattern failure, got: ${result}`, ); @@ -149,7 +263,7 @@ test("no-collapse flips memory-write result flags to false", async () => { const output = print(ast); assert.equal(output.includes("isCollapsible: !1"), true); - assert.equal(output.includes("isMemoryWrite: !1"), true); + assert.equal(output.includes("isMemoryWrite: !!0"), true); assert.equal(noCollapse.verify(output, ast), true); }); @@ -170,7 +284,7 @@ function getCollapseMetadata(H) { } function renderMemoryWriteResult(H, A) { - return { filePath: A, isCollapsible: !1, isMemoryWrite: !1, isSearch: !1, isRead: !1, isREPL: !1 }; + return { filePath: A, isCollapsible: !1, isMemoryWrite: !!0, isSearch: !1, isRead: !1, isREPL: !1 }; } `; const ast = parse(fixtureNoClassification); @@ -378,7 +492,7 @@ function getCollapseMetadata(H) { return null; } function renderMemoryWriteResult(H, A) { - return { filePath: A, isCollapsible: !1, isMemoryWrite: !1, isSearch: !1, isRead: !1, isREPL: !1 }; + return { filePath: A, isCollapsible: !1, isMemoryWrite: !!0, isSearch: !1, isRead: !1, isREPL: !1 }; } `; const ast = parse(literalValueFixture); diff --git a/src/patches/no-collapse.ts b/src/patches/no-collapse.ts index 447b166..fb511f6 100644 --- a/src/patches/no-collapse.ts +++ b/src/patches/no-collapse.ts @@ -25,8 +25,9 @@ import { * * Memory write UI: * - Tool result objects with isCollapsible: !0 + isMemoryWrite: !0 are patched - * to set both to !1 so memory writes render as normal file writes with - * path and diff visible. + * to false so memory writes render as normal file writes with path and + * diff visible. The isMemoryWrite flag uses a stable false-valued AST + * marker so serialized artifacts remain independently verifiable. * * The central result-object factory and its `isCollapsible` property are LEFT INTACT, * so the cache tail scanner still sees `isCollapsible: true` for search/read @@ -36,8 +37,8 @@ import { let memoryWritesPatched = 0; export interface NoCollapseVerificationInventory { - foundMemoryWriteResult: boolean; - foundUnpatchedMemoryWriteResult: boolean; + patchedMemoryWriteResultCount: number; + unpatchedMemoryWriteResultCount: number; foundOriginalGuard: boolean; foundPatchedGuard: boolean; foundClassificationTail: boolean; @@ -46,8 +47,8 @@ export interface NoCollapseVerificationInventory { export function collectNoCollapseVerification( ast: t.File, ): NoCollapseVerificationInventory { - let foundMemoryWriteResult = false; - let foundUnpatchedMemoryWriteResult = false; + let patchedMemoryWriteResultCount = 0; + let unpatchedMemoryWriteResultCount = 0; let foundOriginalGuard = false; let foundPatchedGuard = false; let foundClassificationTail = false; @@ -68,13 +69,15 @@ export function collectNoCollapseVerification( } if (!collapsibleProp || !memoryWriteProp) return; - foundMemoryWriteResult = true; - - if ( - isTrueLike(collapsibleProp.value) && - isTrueLike(memoryWriteProp.value) - ) { - foundUnpatchedMemoryWriteResult = true; + if (isTrueLike(memoryWriteProp.value)) { + unpatchedMemoryWriteResultCount++; + return; + } + if (isPatchedMemoryWriteFalseFlag(memoryWriteProp.value)) { + patchedMemoryWriteResultCount++; + if (!isFalseLike(collapsibleProp.value)) { + unpatchedMemoryWriteResultCount++; + } } }, @@ -143,8 +146,8 @@ export function collectNoCollapseVerification( }); return { - foundMemoryWriteResult, - foundUnpatchedMemoryWriteResult, + patchedMemoryWriteResultCount, + unpatchedMemoryWriteResultCount, foundOriginalGuard, foundPatchedGuard, foundClassificationTail, @@ -168,18 +171,18 @@ export const noCollapse: Patch = { ]; }, - verify: (_code, ast) => { + verify: (_code, ast, context) => { if (!ast) return "Missing AST for no-collapse verification"; const inventory = collectNoCollapseVerification(ast); - if (!inventory.foundMemoryWriteResult) { - return "Memory write result object (isCollapsible + isMemoryWrite) not found"; + if (inventory.unpatchedMemoryWriteResultCount !== 0) { + return "Memory write result object is not fully patched"; } - if (inventory.foundUnpatchedMemoryWriteResult) { - return "Unpatched memory write result object still marks isCollapsible/isMemoryWrite as true"; + if (inventory.patchedMemoryWriteResultCount !== 1) { + return `Expected exactly one patched memory write result marker, found ${inventory.patchedMemoryWriteResultCount}`; } - if (memoryWritesPatched === 0) { - return "Memory write collapsibility was not repointed this run"; + if (context?.phase !== "artifact" && memoryWritesPatched !== 1) { + return `Expected exactly one memory write mutation this run, found ${memoryWritesPatched}`; } if (inventory.foundOriginalGuard) { return "Original collapse-metadata guard (isCollapsible || isREPL) still present"; @@ -218,7 +221,10 @@ function createMemoryWriteUiMutator(): Visitor { if (!isTrueLike(collapsibleProp.value)) return; collapsibleProp.value = t.unaryExpression("!", t.numericLiteral(1)); - memoryWriteProp.value = t.unaryExpression("!", t.numericLiteral(1)); + memoryWriteProp.value = t.unaryExpression( + "!", + t.unaryExpression("!", t.numericLiteral(0)), + ); patched = true; memoryWritesPatched++; }, @@ -234,6 +240,16 @@ function createMemoryWriteUiMutator(): Visitor { }; } +function isPatchedMemoryWriteFalseFlag( + node: t.Node | null | undefined, +): boolean { + return ( + t.isUnaryExpression(node, { operator: "!" }) && + t.isUnaryExpression(node.argument, { operator: "!" }) && + t.isNumericLiteral(node.argument.argument, { value: 0 }) + ); +} + // --------------------------------------------------------------------------- // Collapse UI mutator // --------------------------------------------------------------------------- diff --git a/src/types.ts b/src/types.ts index bc62eb4..e5bf497 100644 --- a/src/types.ts +++ b/src/types.ts @@ -120,6 +120,14 @@ export interface VerificationStageOutcome { /** * A self-contained patch with optional string/AST transformations and verification. */ +export interface PatchVerificationContext { + /** + * `mutation` verifies the patch operation that just ran. `artifact` verifies + * only state that can be recovered from serialized output. + */ + phase: "mutation" | "artifact"; +} + export interface Patch { /** Signature tag name, e.g., "bash-prompt" */ tag: string; @@ -140,7 +148,11 @@ export interface Patch { * Verify patch applied correctly. * Returns true if successful, or a string describing the failure. */ - verify: (code: string, ast?: t.File) => true | string; + verify: ( + code: string, + ast?: t.File, + context?: PatchVerificationContext, + ) => true | string; /** Verify and return structured, code-free semantic evidence in one pass. */ verifyWithWitness?: ( diff --git a/src/verification/verify-cli-anchors.test.ts b/src/verification/verify-cli-anchors.test.ts index 82def17..5c7171a 100644 --- a/src/verification/verify-cli-anchors.test.ts +++ b/src/verification/verify-cli-anchors.test.ts @@ -89,6 +89,34 @@ test("verifyCliAnchors can skip duplicate per-patch verifier pass", async () => } }); +test("verifyCliAnchors runs per-patch verifiers in artifact phase", async () => { + const tempDir = await fs.mkdtemp( + path.join(os.tmpdir(), "anchor-verify-artifact-phase-"), + ); + const patchedCliPath = path.join(tempDir, "patched-cli.js"); + const cleanCliPath = path.join(tempDir, "clean-cli.js"); + const probePatch = allPatches.find( + (patch) => patch.tag === "claudemd-strong", + ); + assert.ok(probePatch); + const originalVerify = probePatch.verify; + let observedPhase: string | undefined; + probePatch.verify = (_code, _ast, context?: { phase?: string }) => { + observedPhase = context?.phase; + return true; + }; + + try { + await fs.writeFile(patchedCliPath, "const marker = 1;", "utf-8"); + await fs.writeFile(cleanCliPath, "const marker = 2;", "utf-8"); + await verifyCliAnchors({ patchedCliPath, cleanCliPath }); + assert.equal(observedPhase, "artifact"); + } finally { + probePatch.verify = originalVerify; + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + test("verifyCliAnchors parses the patched bundle once", async () => { const tempDir = await fs.mkdtemp( path.join(os.tmpdir(), "anchor-verify-single-parse-"), diff --git a/src/verification/verify-cli-anchors.ts b/src/verification/verify-cli-anchors.ts index 2097338..f9036f0 100644 --- a/src/verification/verify-cli-anchors.ts +++ b/src/verification/verify-cli-anchors.ts @@ -385,7 +385,7 @@ function runPatchVerifiers( if (patch.tag === "signature") continue; checksRun++; try { - const result = patch.verify(patchedCode, ast); + const result = patch.verify(patchedCode, ast, { phase: "artifact" }); if (result !== true) { pushFailure( failures,