diff --git a/README.md b/README.md
index 6db14b1..b9bf3d9 100644
--- a/README.md
+++ b/README.md
@@ -65,8 +65,8 @@ Entity / relation / content records can be traced in the parallel `provenance` s
-### 🧪 Lossless by construction
-`__fromPack(pack)` must deep-equal the engine's embedded defaults — extraction bugs surface as path-level diffs, not production surprises.
+### 🧪 Scoped equivalence by construction
+Every extracted literal must be mapped to `__fromPack(pack)` or explicitly ignored with evidence. The configured surface deep-equals engine defaults; runtime/visual behavior needs separate evidence.
@@ -97,8 +97,8 @@ and how entities reference each other.
### 02 · Extract — normalize without losing a byte
-The CLI slices default-data literals straight from the engine source with acorn,
-then your config's `buildPack()` normalizes them: entities get stable slug IDs,
+The CLI slices default-data literals straight from the engine source with acorn as a
+**bootstrap baseline**, then your config's `buildPack()` normalizes them: entities get stable slug IDs,
crawled names become `aliases`, per-stage duplicated copies collapse into
`{a,b}` references, and prose lifts out of templates into `contents`.
@@ -121,8 +121,8 @@ an asset manifest with sha1, entity-identity `sameAs`, and record-level
`SGDataLoader` checks **E1–E16** on mount (dangling refs, illegal enums,
coordinate range, asset registration, scope disambiguation, derivation paths…) and reports
-**W1–W8** honestly. The equivalence test proves `__fromPack(pack)` deep-equals
-the engine defaults; `--verify-hash` catches replaced assets.
+**W1–W8** honestly. Equivalence proves only the configured literal mappings; coverage is explicit.
+`--compare-existing` catches output drift and `--verify-hash` catches replaced assets.
@@ -157,6 +157,24 @@ The human output is deliberately fixed to five sections: **发现的问题**, **
**验证范围**, **剩余风险**, and **下一步**. The JSON and Markdown files are projections of the
same RunReport object, so counts and conclusions cannot drift between CI and developer views.
+### 07 · Agent evaluation — test modifications, not just data
+
+The next layer asks a stricter question: can an AI use these contracts to make a correct component-
+library change? An `AgentTaskManifest` binds instructions, source/input trees, allowed/forbidden
+files, patch limits, grader bytes and runtime/visual evidence requirements. “Hidden” means omitted from the prompt and candidate workspace, not secret from a malicious host process without an OS sandbox. The agent emits a patch only; the runner applies it in a disposable workspace, rechecks the complete tree diff (including `.git/**`), runs task-specific graders from digest-verified per-trial staging copies, and records every artifact digest in TaskRun.
+
+```text
+AgentTaskManifest
+ → patch-only agent
+ → preflight + exact apply + postflight file policy
+ → Data Pack / hidden / command graders
+ → runtime + visual evidence
+ → TaskRun → repeated ExperimentReport
+```
+
+`research/agent-eval/` contains a three-task benchmark and scripted/real-provider experiment specs.
+A scripted run proves the harness only. A real-provider success rate is reported with its valid trial denominator, Wilson 95% interval, actual provider model metadata, tokens, cost, and failure taxonomy. Candidate timeout/crash/evidence failure and post-agent integrity drift remain valid failures; only verified pre-subject infrastructure failures are excluded. It is not generalized into a claim about arbitrary production libraries. TaskRun explicitly records that the portable runner has no OS, network, process, or malicious-agent grader-secrecy sandbox.
+
---
## Quick Start
@@ -177,10 +195,12 @@ ln -s "$PWD/sg-data-pack" ~/.claude/skills/sg-data-pack # Claude Code
```bash
SK=~/.zcode/skills/sg-data-pack/scripts/sg-data-pack
-node "$SK" extract # extract + validate + equivalence test
-node "$SK" extract --check # validate only (regression)
-node "$SK" validate --strict --verify-hash
-node "$SK" rules [--strict]
+node "$SK" extract # bootstrap from source literals; refuses divergent existing data.json
+node "$SK" extract --check # source-equivalence only; does not read current data.json
+node "$SK" extract --compare-existing # compare fresh pack with current data.json
+node "$SK" compile [--domain-schema fragment.json] [--check] # reviewed data.json -> data.js + schema; explicit custom domain contract
+node "$SK" validate --strict --verify-hash [--asset-root dir]
+node "$SK" rules [--strict] [--rule id]
node "$SK" diff [--json]
node "$SK" templatize [--out dir]
node "$SK" alias-candidates
@@ -192,6 +212,11 @@ node "$SK" report [--config extract.config.js] [--baseline old-data.jso
[--review review-report.json] [--audit candidate-audit.json] [--strict] \
[--verify-hash] [--out report/] [--json]
node "$SK" types [--out data-types.d.ts] [--name PackName]
+node "$SK" task validate [--json]
+node "$SK" task run --agent-command --agent-arg '' --artifacts [--json]
+node "$SK" grade [--artifacts dir] [--json]
+node "$SK" evidence [--json]
+node "$SK" experiment --out [--json]
node "$SK" loader # print runtime-validator path (copy into a library's lib/src/)
node "$SK" schema # print contract-schema path
```
@@ -203,16 +228,20 @@ node "$SK" schema # print contract-schema path
### Recommended product workflow
```bash
-# 1. Create or refresh the canonical pack
+# 1. Bootstrap the canonical pack once; later detect, do not overwrite, reviewed evolution
node "$SK" extract path/to/extract.config.js
+node "$SK" extract path/to/extract.config.js --compare-existing
-# 2. Produce one human + CI hand-off
+# 2. After reviewing/editing data.json, regenerate browser/schema artifacts from that canonical source
+node "$SK" compile path/to/library/lib/data/data.json
+
+# 3. Produce one human + CI hand-off
node "$SK" report path/to/library \
--config path/to/extract.config.js \
--verify-hash \
--out report/
-# 3. For evolution or recrawl work, bind the related evidence
+# 4. For evolution or recrawl work, bind the related evidence
node "$SK" report path/to/library \
--baseline old-data.json \
--review review/review-report.json \
@@ -310,6 +339,12 @@ Full contract: [`references/data-pack-contract.md`](references/data-pack-contrac
| [`references/data-pack-contract.md`](references/data-pack-contract.md) | Data Pack v1.3 field-level contract |
| [`references/review-candidate-contract.md`](references/review-candidate-contract.md) | Candidate-ready recrawl reports, explicit decisions, and audit contract |
| [`references/run-report-contract.md`](references/run-report-contract.md) | Unified Library Evolution Report, coverage, risks, and exit codes |
+| [`references/agent-task-contract.md`](references/agent-task-contract.md) | Content-bound task, source/input, file-policy and grader contract |
+| [`references/patch-execution-contract.md`](references/patch-execution-contract.md) | Patch-only execution, postflight file enforcement and PatchAudit |
+| [`references/grader-contract.md`](references/grader-contract.md) | Task-specific checks, score and GradeReport |
+| [`references/task-run-contract.md`](references/task-run-contract.md) | One agent execution and artifact audit |
+| [`references/runtime-visual-evidence-contract.md`](references/runtime-visual-evidence-contract.md) | Runtime/visual producer evidence and recomputed gates |
+| [`references/agent-experiment-contract.md`](references/agent-experiment-contract.md) | Repeated TaskRuns, success rate, Wilson interval and failure taxonomy |
| [`references/extraction-config.md`](references/extraction-config.md) | Config guide + three real-world patterns |
| [`references/engine-integration.md`](references/engine-integration.md) | Engine-patch standard template |
| [`assets/extract.config.template.js`](assets/extract.config.template.js) | Annotated config template for a new library |
@@ -323,7 +358,13 @@ the TypeScript generator, and cross-artifact version consistency:
node --test tests/*.test.js
```
-The same suite runs in CI via `.github/workflows/smoke.yml`. Three reproducible v1.3 synthetic pilots and their machine-readable result live under `research/`; rerun them with `node research/run-v1.3-pilots.js`. The suite also contains a hermetic real-engine integration fixture for the Qinshihuang event graph at `tests/fixtures/integration/qinshihuang-0716-ts/`, covering external HTML JSON extraction, deep equivalence, committed asset baselines plus tamper detection, library rules, generated types, and real derivation impact.
+The same suite runs in CI via `.github/workflows/smoke.yml`. Three reproducible v1.3 synthetic pilots and their machine-readable result live under `research/`; rerun them with `node research/run-v1.3-pilots.js`. The suite also contains a hermetic real-engine integration fixture for the Qinshihuang event graph at `tests/fixtures/integration/qinshihuang-0716-ts/`, covering external HTML JSON extraction, configured equivalence, committed asset baselines plus tamper detection, library rules, generated types, and real derivation impact.
+
+Agent evaluation fixtures live under `research/agent-eval/`: three content-bound tasks, hidden graders,
+patch-only providers, a real headless-Chrome/Pillow evidence producer, and scripted/AI experiment specs.
+Regenerate their task/experiment ids with `node research/agent-eval/build-fixtures.js`. Committed result
+reports state their narrow benchmark scope and preserve failures rather than presenting a green test
+suite as proof of general AI capability.
## License
diff --git a/SKILL.md b/SKILL.md
index a1c8d01..c371695 100644
--- a/SKILL.md
+++ b/SKILL.md
@@ -20,10 +20,12 @@ A Data Pack is the **single entry point** for a component library's business dat
```bash
SK=~/.zcode/skills/sg-data-pack/scripts/sg-data-pack # or the codex/claude install path
-node "$SK" extract # extract + validate + equivalence test; writes lib/data/{data.json,data.js,data.schema.json}
-node "$SK" extract --check # validate + equivalence only (no writes)
-node "$SK" validate [--strict] [--verify-hash] # standalone validation (--verify-hash detects replaced assets)
-node "$SK" rules [--strict] # execute library-level data rules (data-rules.json)
+node "$SK" extract # bootstrap from source literals; divergent existing data.json requires --force
+node "$SK" extract --check # source-equivalence only; does not read current data.json
+node "$SK" extract --compare-existing # fresh in-memory pack vs current data.json
+node "$SK" compile [--domain-schema fragment.json] [--check] # preserve explicit custom domain schema
+node "$SK" validate [--strict] [--verify-hash] [--asset-root dir]
+node "$SK" rules [--strict] [--rule id] # execute all or one library rule
node "$SK" diff [--json] # structural diff between two packs (evolution / recrawl review)
node "$SK" templatize [--out dir] # derive item template from repeated HTML instances (collection pages)
node "$SK" alias-candidates # rank unresolved crawled names
@@ -31,6 +33,11 @@ node "$SK" recrawl-skeleton [--out dir] # generate c
node "$SK" data-surface-import [--out report.json] [--allow-review-required] # read-only interface handoff; never generate a Data Pack
node "$SK" candidate --records --out # apply explicit Review Decisions
node "$SK" report [--config config.js] [--baseline old.json] [--review review-report.json] [--audit candidate-audit.json] [--strict] [--verify-hash] [--out report/] [--json] # unified user-facing report
+node "$SK" task validate [--json] # validate source/input/grader-bound AgentTaskManifest
+node "$SK" task run --agent-command --agent-arg '' --artifacts [--json] # patch-only disposable TaskRun
+node "$SK" grade [--artifacts dir] [--json]
+node "$SK" evidence [--json]
+node "$SK" experiment --out [--json]
node "$SK" types [--out file.d.ts] [--name N] # generate TypeScript declarations
node "$SK" loader # print runtime-validator path (copy into the library's lib/src/)
node "$SK" schema # print contract schema path
@@ -84,9 +91,10 @@ cp "$(node "$SK" loader)" /lib/src/sg-data-loader.js
node "$SK" extract # must be fully green: 0 validation errors, equivalence passed
```
-The equivalence test is the **losslessness guarantee**: `__fromPack(pack)` must deep-equal the
-embedded defaults. Common failure causes: missing fields in fromPack / key-order changes /
-undeduplicated duplicates — fix item by item using the reported diff paths.
+The equivalence test is a **configured-surface guarantee**: every extracted literal must map to an
+own field returned by `__fromPack(pack)` or be explicitly ignored with an evidence-backed reason.
+It does not validate the current disk `data.json`, DOM mount, renderer constants, or pixels. Use
+`--compare-existing`, TaskRun runtime evidence, and visual evidence for those independent claims.
### 5. Integrate + regress
@@ -128,6 +136,24 @@ remaining risks, and next steps. Use `--json` for CI; JSON stdout contains only
`NOT_ASSESSED` means no evidence was supplied, never “passed”. The report is read-only and binds
consumed inputs by SHA-256. Full field and exit-code semantics are in `references/run-report-contract.md`.
+### 8. Evaluate an AI modification under contract
+
+Use an AgentTaskManifest only after the Data Pack, file policy, hidden grader and required runtime/
+visual scenarios are explicit. “Hidden” means absent from the prompt/workspace, not secret from an unsandboxed malicious local process. The agent must emit a unified diff; it must not edit source or the disposable workspace directly. TaskRun rechecks the complete tree including `.git/**`, revalidates task/adapter/grader/tool digests across stages, grades from per-trial verified staging copies, and preserves PatchAudit, GradeReport and evidence artifacts. Missing OS/filesystem/network/process isolation remains visible.
+
+```bash
+node "$SK" task validate path/to/task.json
+node "$SK" task run path/to/task.json --agent-command \
+ --agent-arg '{workspace}' --agent-arg '{patch}' --agent-arg '{prompt}' \
+ --artifacts /tmp/task-run
+node "$SK" experiment research/agent-eval/experiment-claude.json --out /tmp/agent-experiment
+```
+
+Never treat a scripted provider as AI evidence. Never exclude invalid/infra trials silently. Report
+valid denominator, Wilson interval, actual provider model metadata, token/cost evidence, failure
+taxonomy, and which task subset had runtime/visual evidence. See the agent-task, patch-execution,
+grader, task-run, runtime/visual, and agent-experiment contracts in `references/`.
+
## Rule Cheat Sheet (enforced by the validator)
| Rule | Meaning |
@@ -158,6 +184,12 @@ Crawl output may only consist of: `data.json` + `assets/` + the alias table. Req
- `references/data-surface-manifest-import.md` — read-only Data Surface Manifest handoff and review gate
- `references/review-candidate-contract.md` — candidate-ready reports, explicit decisions, and audit sidecars
- `references/run-report-contract.md` — unified Library Evolution Report, coverage, risks, and exit codes
+- `references/agent-task-contract.md` — content-bound task/source/input/file/grader manifest
+- `references/patch-execution-contract.md` — patch-only application, file policy, PatchAudit and capability limits
+- `references/grader-contract.md` — task-specific checks, score and GradeReport
+- `references/task-run-contract.md` — one agent execution and all artifact digests
+- `references/runtime-visual-evidence-contract.md` — runtime/visual evidence and recomputed gates
+- `references/agent-experiment-contract.md` — repetitions, success rate, Wilson interval and failure taxonomy
- `references/extraction-config.md` — extraction-config guide with three typical patterns
- `references/engine-integration.md` — engine-patch standard template (copy-paste grade)
- `references/data-rules-guide.md` — library-level feature rules: format, check convention, evidence discipline
diff --git a/assets/extract.config.template.js b/assets/extract.config.template.js
index 9709b26..9902fe2 100644
--- a/assets/extract.config.template.js
+++ b/assets/extract.config.template.js
@@ -14,8 +14,9 @@ module.exports = {
libDir: LIB_DIR,
engineFile: 'lib/src/your-lib.js',
globalName: 'YourLibrary', // engine global name (global.X = {mount, create})
+ assetDir: 'lib/assets', // physical asset root relative to libDir; independent from meta.assetBase
- // Default-data literal slicing (losslessness baseline). Pattern match end = expression start.
+ // Default-data literal slicing (equivalence baseline). Pattern match end = expression start.
literals: [
{ key: 'chars', pattern: /var chars = \(options && options\.chars\) \|\|/, ctx: { IMG: '../assets/' } },
// { key: 'edges', pattern: /var edges = \(options && options\.edges\) \|\|/ },
@@ -81,11 +82,13 @@ module.exports = {
return pack;
},
- // Losslessness check: fromPack(pack)[from] deep-equals defaults[lit]
+ // Configured equivalence check: fromPack(pack)[from] deep-equals defaults[lit]
equivalence: [
{ lit: 'chars', from: 'chars' },
// { lit: 'edges', from: 'allEdges' },
],
+ // Every literal must be mapped above or explicitly ignored with an evidence-backed reason:
+ // equivalenceIgnore: [{ lit: 'rendererDefaults', reason: 'Renderer-only; covered by visual regression VR-1' }],
// sortKeys: { edges: (x, y) => edgeKey(x).localeCompare(edgeKey(y)) },
diff --git a/references/agent-experiment-contract.md b/references/agent-experiment-contract.md
new file mode 100644
index 0000000..fd4b141
--- /dev/null
+++ b/references/agent-experiment-contract.md
@@ -0,0 +1,24 @@
+# Agent success-rate ExperimentSpec and ExperimentReport v1.0
+
+An experiment runs a fixed task set repeatedly from fresh immutable baselines:
+
+```bash
+node scripts/sg-data-pack experiment research/agent-eval/experiment-claude.json \
+ --out /tmp/sg-agent-experiment
+```
+
+`ExperimentSpec` binds agent kind/provider/model label, argv, the actual adapter path and byte digest, ordered task manifest paths/taskIds/manifest digests, repetitions and seed through `experimentId`. Each task × repetition receives a new disposable workspace and its own TaskRun artifact directory. The experiment rechecks spec, adapter, task manifest/taskId, source, grader bundle, and tool integrity before/after every trial rather than trusting one initialization check.
+
+The report preserves:
+
+- total, valid, invalid and infrastructure-error trial counts;
+- pass rate over valid trials and Wilson 95% interval;
+- stage rates for agent completion, patch apply, file-policy compliance, graders, runtime and visual;
+- verdict failure taxonomy;
+- mean/median duration;
+- provider-reported actual models, input/output/cache tokens and cost;
+- a reference to every TaskRun.
+
+Invalid input and pre-subject infrastructure failures are reported separately rather than silently removed or counted as AI task failures. Candidate timeout/crash/evidence failure and any post-agent integrity drift remain valid failures. A simultaneous candidate failure takes precedence over infra so a failing trial cannot disappear from the denominator. Runtime/visual denominators include every valid trial whose task declares those checks, including patch/policy/grader precondition failures that never reached evidence collection.
+
+A scripted provider proves the harness only (`harness-only`). A real AI provider can support an initial task-contract claim, but a small synthetic benchmark is not evidence of general component library autonomy. `ai-task-contract-not-demonstrated` means no valid AI trial passed. `ai-static-contract-only` means at least one valid AI trial passed but required runtime/visual coverage was not fully successful. `ai-task-contract-with-runtime-visual-subset` means at least one valid trial passed and every valid trial in the declared browser-evidence subset passed its runtime/visual checks; it is deliberately narrower than a claim about arbitrary production libraries.
diff --git a/references/agent-task-contract.md b/references/agent-task-contract.md
new file mode 100644
index 0000000..7d05863
--- /dev/null
+++ b/references/agent-task-contract.md
@@ -0,0 +1,94 @@
+# AgentTaskManifest v1 contract
+
+`AgentTaskManifest` is a declarative, content-addressed description of one agent task. The machine-readable contract is [`scripts/lib/agent-task.schema.json`](../scripts/lib/agent-task.schema.json); the read-only implementation is [`scripts/lib/agent-task-manifest.js`](../scripts/lib/agent-task-manifest.js).
+
+The current version is exactly `taskVersion: "1.0"`. The validator fails closed: unknown fields, malformed paths, missing digests, duplicate identifiers/patterns, and resource-policy violations are errors, not warnings.
+
+## Shape
+
+A manifest has exactly these top-level fields:
+
+- `taskVersion`: `"1.0"`;
+- `taskId`: `sha256:<64 lowercase hex>`;
+- `instructions`: `{ text, sha256 }`, where `sha256` hashes the UTF-8 bytes of `text`;
+- `source`: `{ root, revision, treeSha256 }`;
+- `inputs`: entries `{ id, path, role, sha256 }`;
+- `filePolicy`: `allowed` rules, `forbidden` patterns, `denyPrecedence: true`, `allowSymlinks: false`, `allowHardlinks: false`, `maxChangedFiles`, and `maxPatchBytes`;
+- `patchPolicy`: `{ format: "unified-diff", fuzz: 0, allowBinary: false }`;
+- `graders`: entries `{ id, spec, sha256, treeSha256, weight }`; the spec bytes and complete hidden grader/reference directory are content-bound;
+- `evidence`: `{ runtime: [], visual: [] }` arrays (their contents are evidence descriptors owned by the caller);
+- `execution`: `{ timeoutMs, network: "off", maxOutputBytes }`.
+
+`id` values are unique within their respective arrays (`inputs` and `graders`). `filePolicy.allowed[].pattern` and `filePolicy.forbidden[]` do not repeat. `allowed[].operations` contains one or more distinct operations from `add`, `modify`, and `delete`. A forbidden match wins over an allowed match because `denyPrecedence` is mandatory `true`.
+
+All digest fields use the repository-wide prefixed form `sha256:`. `sha256` means SHA-256 over bytes, not a hexadecimal string that has first been re-encoded.
+
+## Canonical JSON and task ID
+
+Canonical JSON is compact `JSON.stringify` output after recursively sorting keys of every JSON object. Array order is preserved. There is no trailing newline, whitespace normalization, number conversion, or array sorting. The canonical material for the task ID is the complete manifest with the `taskId` property removed. Therefore:
+
+```text
+taskId = sha256(UTF-8(canonicalJson(manifest without taskId)))
+```
+
+`computeTaskId(manifest)` implements this rule. It is useful when authoring a manifest, but `validateTaskManifest` still checks that the supplied ID is present and matches; computing a new ID does not make an otherwise invalid manifest valid.
+
+The implementation also exposes `canonicalJson`, `canonicalize`, `sha256`, and `computeTreeSha256` for producers and tests. These functions do not write files.
+
+## Path rules and resolution
+
+Manifest paths are POSIX-style, relative, non-empty paths. Concrete `source.root` and `inputs[].path` values are not globs; only the `filePolicy` pattern fields may contain glob metacharacters. They must not contain:
+
+- a leading `/` or a Windows drive prefix such as `C:`;
+- a backslash (`\\`), NUL, or empty path components (`//`);
+- `.` or `..` path components (the source root may be `"."` to mean its task-file directory, but other dot components are rejected).
+
+`source.root` is relative to the **task file's directory** when `taskFile` is supplied. `inputs[].path`, `filePolicy.allowed[].pattern`, and `filePolicy.forbidden[]` are relative to the resolved source root. Thus for `tasks/foo/task.json` and `source.root: "../../project"`, an input `src/a.js` means `../../project/src/a.js` from the task file directory. The path is resolved and must remain within that source root when files or the tree are verified.
+
+For direct programmatic validation, `sourceRoot` may be supplied. It must be the already resolved source-root directory and must equal the resolution of `source.root` if `taskFile` is also supplied. `taskFile` takes precedence for resolving `source.root`; `sourceRoot` is then a consistency check. `readTaskManifest(file)` always passes the manifest file as `taskFile`.
+
+A path is never resolved relative to the process working directory merely because the manifest omits `taskFile`. This avoids making a manifest's meaning depend on the caller's current directory.
+
+## Source and file verification
+
+`source.revision` is the producer's immutable source revision identifier (for example, a commit or checkout label). The v1 helper treats it as an opaque non-empty string; it does not invoke Git.
+
+`source.treeSha256` binds a deterministic filesystem-tree description. `computeTreeSha256(root)` walks the complete tree read-only, sorts names by their UTF-8 bytes, and hashes canonical JSON entries. Entries bind path, kind, mode, and regular-file bytes; security snapshots include `.git/**` rather than applying reporting excludes. Symlinks and hardlinks are rejected by default, matching the manifest policy. The tree helper can explicitly opt into them for independent tooling, but a v1 manifest cannot: `allowSymlinks` and `allowHardlinks` must remain false.
+
+- `validateTaskManifest(manifest, { verifyFiles: true, taskFile, sourceRoot })` reads each input and compares its byte digest. It rejects missing files, non-regular files, symlinks, hardlinks, and files outside the source root.
+- `validateTaskManifest(manifest, { verifyTree: true, taskFile, sourceRoot })` computes and compares `source.treeSha256` and applies the same link policy to every tree entry.
+- These checks are opt-in because the pure shape/identity validation does not need filesystem access. All checks are read-only.
+
+## Resource and patch gates
+
+The implementation applies finite bounds in addition to non-negative/integer checks:
+
+- `execution.timeoutMs`: `1..86400000` (24 hours);
+- `execution.maxOutputBytes`: `1..1073741824` (1 GiB);
+- `filePolicy.maxChangedFiles`: `0..1000000`;
+- `filePolicy.maxPatchBytes`: `0..1073741824` (1 GiB).
+
+The task requests offline candidate/grader execution. Only unified diffs with zero fuzz and no binary patch content are accepted. The portable v1 runner enforces patch and filesystem postconditions, rechecks immutable task/grader/tool bindings after untrusted stages, and runs candidate extraction code in a killable child worker. It does **not** claim OS-level filesystem, network, or process isolation; TaskRun records those capabilities as false. In particular, “hidden grader” means omitted from the agent prompt and disposable candidate workspace, not cryptographically unreadable to a malicious local process on an unsandboxed host. A remote AI provider is an external control-plane dependency and is reported separately from candidate execution.
+
+## API
+
+```js
+const {
+ validateTaskManifest,
+ computeTaskId,
+ readTaskManifest,
+} = require('../scripts/lib/agent-task-manifest.js');
+
+const result = validateTaskManifest(manifest, {
+ taskFile: '/absolute/path/to/task.json',
+ sourceRoot: '/absolute/path/to/source',
+ verifyFiles: true,
+ verifyTree: true,
+});
+// { valid, issues: [{ path, message }], expectedTaskId, sourceRoot, ... }
+
+const taskId = computeTaskId(manifest);
+const loaded = readTaskManifest('/absolute/path/to/task.json'); // throws on invalid JSON/manifest
+```
+
+`readTaskManifest` parses one JSON file and returns the parsed manifest. It never rewrites, normalizes, or updates it. Invalid input throws an error with code `INVALID_AGENT_TASK_MANIFEST` and structured `issues`; unreadable or malformed JSON throws `AGENT_TASK_MANIFEST_READ_ERROR`.
diff --git a/references/data-rules-guide.md b/references/data-rules-guide.md
index 096702d..eb48ddb 100644
--- a/references/data-rules-guide.md
+++ b/references/data-rules-guide.md
@@ -6,8 +6,10 @@ rules from the pack + engine + assets. Two artifacts, both living in `lib/data/`
- `data-rules.json` — structured rules (machine-executable / machine-searchable)
- `DATA-GUIDE.md` — human contributor guide generated from the rules
-Validate any rules file with: `node scripts/sg-data-pack rules [--strict]`
-(hard-rule failures exit 1; soft failures are warnings unless --strict).
+Validate any rules file with: `node scripts/sg-data-pack rules [--strict] [--rule ]`
+(hard-rule failures exit 1; soft failures are warnings unless --strict). `--rule ` selects one
+known rule after the complete rules document has been structurally validated; unknown or repeated
+options/ids are usage errors (exit 2).
## data-rules.json Format (rulesVersion 1.0)
@@ -29,7 +31,7 @@ Validate any rules file with: `node scripts/sg-data-pack rules [--stric
{
"id": "-", // e.g. sdd-hero-big-exclusive
"level": "hard | soft",
- "subject": "anchor path, e.g. entities.*.avatar / stages.*.entities / assets.*",
+ "subject": "human locator string, e.g. entities.*.avatar / stages.*.entities / assets.*",
"rule": "the rule text (concrete, actionable; no vague words like 'try to' / 'appropriate')",
"rationale": "why — cite rendering logic or data evidence",
"evidence": "measured evidence, e.g. '15/15 avatars are .png; 5 sampled at 160px width'",
@@ -42,12 +44,18 @@ Validate any rules file with: `node scripts/sg-data-pack rules [--stric
}
```
-## check Expression Convention (hard requirement)
+## subject, scope, and check conventions (hard requirement)
-- `check` is a **JS expression string**, evaluated with pack sections in scope:
+- `subject` is an **unparsed human locator string**. It tells a contributor where to look (for
+ example `entities.*.avatar` or `stages.*.entities`); the evaluator does not resolve it,
+ interpret wildcards, or use it to select records.
+- `check` is a **JavaScript expression string**, evaluated with pack sections in scope:
`entities / relations / stages / contents / domain / assets / aliases / relationTypes /
- attributeTypes / kindNameFields / meta / sameAs / provenance / pack`
-- Must evaluate to a truthy/falsy value. If you cannot write an executable expression, use `null`.
+ heroRelTypes / attributeTypes / attributeSources / kindNameFields / meta / sameAs /
+ provenance / derivations / pack`.
+ This scope must stay synchronized with the executor whenever a pack root becomes available
+ to rules. In particular, `heroRelTypes`, `attributeSources`, and `derivations` are supported.
+- The expression must evaluate to a truthy/falsy value. If you cannot write an executable expression, use `null`.
- **Pseudo-DSL is forbidden** (`all(entities.*, e -> ...)` does not run). Write real JS:
`Object.values(entities).every(e => ...)`
- After writing, **evaluate every check against the real data.json with node — all must pass**.
diff --git a/references/data-surface-consumer-profile.md b/references/data-surface-consumer-profile.md
new file mode 100644
index 0000000..b4ba0d8
--- /dev/null
+++ b/references/data-surface-consumer-profile.md
@@ -0,0 +1,17 @@
+# Data Surface consumer profile
+
+This file is a **local consumer profile**, not the upstream Data Surface Manifest schema. The `ui-dismantler` producer contract remains authoritative. The profile records the minimum boundary checks performed by `sg-data-pack`; it must not be copied upstream or used to generate a Data Pack.
+
+## Import boundary
+
+The importer accepts a Manifest with `schemaVersion: "1.0"`, `kind: "data-surface-manifest"`, producer identity hashes, library metadata, surfaces, unresolved/review state, and metrics. A surface describes an interface evidence boundary: owner, source, shape, fields, consumers, injection, references, unresolved notices, and optional evidence.
+
+`surface.reviewRequired`, when supplied, is strictly a JSON boolean. The top-level `reviewRequired` is required and is also strictly boolean. Raw static business values are rejected.
+
+## Data Pack separation
+
+No Data Pack v1.3 root section may appear in a handoff. This includes `meta`, `kindNameFields`, `sameAs`, `provenance`, `entities`, `aliases`, `relationTypes`, `heroRelTypes`, `relations`, `stages`, `attributeTypes`, `attributeSources`, `contents`, `domain`, `assets`, and `derivations`; the legacy `adapters` field is rejected as well. This is a consumer-side boundary guard, not a replacement for the producer schema.
+
+## Evidence and digest
+
+The importer preserves each surface's evidence in the audit report. `source.digest` is SHA-256 over the complete Manifest after recursively sorting object keys while retaining array order. Therefore any evidence, review, unresolved, metric, identity, owner, or field change changes the digest; object key ordering alone does not.
diff --git a/references/data-surface-manifest-import.md b/references/data-surface-manifest-import.md
index effacfa..6f2623c 100644
--- a/references/data-surface-manifest-import.md
+++ b/references/data-surface-manifest-import.md
@@ -1,6 +1,6 @@
# Data Surface Manifest import
-`ui-dismantler` owns the Data Surface Manifest contract. `sg-data-pack` consumes it read-only as a component-data handoff; it does not copy the contract schema or reinterpret a Manifest as a Data Pack.
+`ui-dismantler` owns the Data Surface Manifest contract. `sg-data-pack` consumes it read-only as a component-data handoff; it does not copy the contract schema or reinterpret a Manifest as a Data Pack. The local [consumer profile](data-surface-consumer-profile.md) describes only the checks this importer needs and is **not** an authoritative upstream schema. When the profile and the producer contract differ, the `ui-dismantler` contract is authoritative and this consumer must be updated deliberately.
```bash
node scripts/sg-data-pack data-surface-import \
@@ -17,6 +17,8 @@ The importer accepts only interface evidence:
- injection and reference boundaries;
- unresolved and review state.
-It rejects raw static values and top-level Data Pack fields such as `entities`, `aliases`, `relations`, `stages`, `contents`, or `adapters`.
+It rejects raw static values and every Data Pack v1.3-only root section (`meta`, `kindNameFields`, `sameAs`, `provenance`, `entities`, `aliases`, relation registries/edges, stages, attribute registries/sources, `contents`, `domain`, `assets`, and `derivations`), plus the legacy `adapters` field. `surface.reviewRequired`, when present, must be an actual JSON boolean; strings and numeric stand-ins are invalid.
-Import output is a `data-surface-import-report` with an explicit `dataPackGenerationAllowed` gate. Business entity modeling starts only after the report is `ready`; a review-required report remains evidence, not a generation input.
+Import output is a `data-surface-import-report` with an explicit `dataPackGenerationAllowed` gate. Business entity modeling starts only after the report is `ready`; a review-required report remains evidence, not a generation input. Each surface's `evidence` is preserved, and `source.digest` is the SHA-256 of the complete canonical Manifest (recursively key-sorted JSON), not a projection of selected identity or metrics fields.
+
+The CLI rejects unknown or repeated options. `--out` may not resolve to the input Manifest through an identical canonical path, symlink, or inode/hardlink, and report replacement uses a same-directory temporary file followed by an atomic rename.
diff --git a/references/engine-integration.md b/references/engine-integration.md
index 159c32c..5d70a63 100644
--- a/references/engine-integration.md
+++ b/references/engine-integration.md
@@ -5,7 +5,11 @@ must be byte-identical when `options.data` is absent**.
Technique: convert the pack into the legacy options shape, reusing the existing
`(options && options.x) || defaults` paths.
-## Standard Patch (3 spots)
+## Integration Shell (3 spots)
+
+The injection shell is universal; the body of `__fromPack` is an adapter for the library's legacy
+shape. The first example below is the **entity-graph adapter**, not a mandatory Data Pack shape.
+Plain collections and DOM JSON-script pages should use the dedicated adapters later in this guide.
### ① Insert at the top of the engine IIFE (before mount)
@@ -98,6 +102,68 @@ var heroRel = (options && options.heroRel) || { ... };
global.MyLib = { mount: mount, create: create, __fromPack: __fromPack };
```
+## Pattern C: DOM JSON-script adapter
+
+When the legacy data is the whole options payload in a DOM JSON script, preserve that parse path as
+the no-pack fallback and make the injected path additive:
+
+```js
+function __fromPack(pack) {
+ // Rebuild the exact legacy JSON object. Keep this adapter library-specific.
+ return {
+ items: Object.keys(pack.entities || {}).map(function (id) {
+ var entity = pack.entities[id];
+ return { id: id, label: entity.label, image: entity.image };
+ }),
+ categories: (pack.domain && pack.domain.categories) || []
+ };
+}
+
+function __resolveDataOptions(options) {
+ if (!options || !options.data) return options;
+ if (typeof SGDataLoader !== 'undefined') SGDataLoader.assertValid(options.data);
+ var legacyOptions = __fromPack(options.data);
+ var merged = {};
+ Object.keys(options).forEach(function (key) { if (key !== 'data') merged[key] = options[key]; });
+ Object.keys(legacyOptions).forEach(function (key) { merged[key] = legacyOptions[key]; });
+ return merged;
+}
+
+function readDomDefaults() {
+ var element = document.getElementById('sg-data');
+ if (!element || !element.textContent) throw new Error('Missing #sg-data JSON defaults');
+ return JSON.parse(element.textContent);
+}
+
+function mount(root, options) {
+ options = __resolveDataOptions(options);
+ var data = options && options.items ? options : readDomDefaults();
+ // Original rendering logic continues unchanged and consumes `data`.
+}
+```
+
+The extraction equivalence mapping compares the sliced JSON object with `__fromPack(pack)`. It does
+**not** execute `mount`, parse the live DOM, collect browser errors, or compare pixels. Those are
+separate runtime/visual assurances and remain `NOT_ASSESSED` until evidence is supplied.
+
+## Plain collection adapter
+
+A collection library does not need graph-shaped variables:
+
+```js
+function __fromPack(pack) {
+ return {
+ cards: Object.keys(pack.entities || {}).map(function (id) {
+ var entity = pack.entities[id];
+ return { key: id, title: entity.title, summary: entity.summary };
+ })
+ };
+}
+```
+
+Return the exact legacy collection names consumed by the engine. Do not manufacture `chars`,
+`allEdges`, or `storyModules` unless those are real engine inputs.
+
## Variants
| Scenario | Approach |
@@ -113,7 +179,7 @@ global.MyLib = { mount: mount, create: create, __fromPack: __fromPack };
1. **Zero rendering-logic changes** — all edits live at the data entry point
2. **Byte-identical behavior without options.data** — `__resolveDataOptions` is a pass-through
-3. **Equivalence is a hard gate** — `__fromPack(pack)` deep-equals the embedded defaults; never skip it
+3. **Configured equivalence is a hard gate** — every extracted literal is mapped or explicitly ignored with evidence; mapped `__fromPack(pack)` fields deep-equal the embedded defaults
4. **No silent data anomalies** — dangling refs/duplicates in source data are reported, never filtered away
5. The loader must be copied to `lib/src/sg-data-loader.js` (`node scripts/sg-data-pack loader` prints its path)
diff --git a/references/extraction-config.md b/references/extraction-config.md
index 0999054..b92ffd5 100644
--- a/references/extraction-config.md
+++ b/references/extraction-config.md
@@ -1,9 +1,13 @@
# Extraction Config Guide
A config is a CommonJS module that tells the CLI where the defaults live in the engine, how to
-convert them into a spec-compliant pack, and how to verify losslessness.
+convert them into a spec-compliant pack, and how to verify the configured equivalence surface.
Template: `assets/extract.config.template.js`.
+`extract` is a source-to-pack bootstrap operation. Its baseline is the configured engine/HTML literals,
+not an existing `lib/data/data.json`. Use `--compare-existing` to compare the fresh in-memory pack with
+the current output. A divergent existing pack is never overwritten unless `--force` is explicit.
+
## Full Field Reference
```js
@@ -13,8 +17,9 @@ module.exports = {
engineFile: 'lib/src/my-lib.js', // engine file (relative to libDir)
globalName: 'MyLibrary', // engine global name (read after requiring the engine)
schemaVersion: '1.3', // optional, default 1.3
+ assetDir: 'lib/assets', // optional physical asset root, relative to libDir
- literals: [ // default-data literal slicing (losslessness baseline)
+ literals: [ // default-data literal slicing (equivalence baseline)
// JS literal: pattern matches up to and including `||` or `=`; acorn parses the expression from there
{ key: 'chars', pattern: /var chars = \(options && options\.chars\) \|\|/, ctx: { IMG: '../assets/' } },
// JSON embedded in HTML: `file` targets another file; with json:true, reads to the next
@@ -25,9 +30,11 @@ module.exports = {
buildPack(defaults) { /* defaults. -> pack */ },
- equivalence: [ // losslessness check: fromPack(pack)[from] vs defaults[lit] (deep-equal)
+ equivalence: [ // fromPack(pack)[from] vs defaults[lit] (deep-equal)
{ lit: 'chars', from: 'chars' }
],
+ // Every literal must be mapped or explicitly ignored with evidence:
+ // equivalenceIgnore: [{ lit: 'rendererDefaults', reason: 'Renderer-only; covered by visual regression VR-1' }],
sortKeys: { chars: (x, y) => 0 }, // optional: sort arrays before comparing (order-insensitive)
domainChecks(pack) { return { errors: [], warnings: [] }; }, // optional: library-level domain validation
domainSchema: { type: 'object' } // optional: domain extension written into data.schema.json
@@ -42,7 +49,11 @@ module.exports = {
into a cooked string), then extract with regex inside buildPack. In minified files, comma-chained
declarations make acorn overrun — split the declarations into separate statements first
- The equivalence test `require`s the engine file: the engine must be an IIFE/UMD that only defines
- functions on load and never touches the DOM (all sg-* engines satisfy this)
+ functions on load and never touches the DOM. This is a trusted-code requirement, not a sandbox.
+- `assetDir` controls the physical filesystem root. `meta.assetBase` remains the logical prefix used
+ to resolve bare asset references into manifest keys; the two settings are intentionally separate.
+- Equivalence only proves the declared mappings. Renderer constants and actual DOM mount/visual behavior
+ require runtime or visual evidence and must not be described as passed by this check.
## buildPack Conversion Patterns
diff --git a/references/grader-contract.md b/references/grader-contract.md
new file mode 100644
index 0000000..18a046d
--- /dev/null
+++ b/references/grader-contract.md
@@ -0,0 +1,17 @@
+# Task-specific GraderSpec and GradeReport v1.0
+
+A grader is a trusted, task-bound contract outside the agent workspace. `AgentTaskManifest.graders[]` binds both the grader spec bytes and the complete grader directory tree by SHA-256, including hidden assertions and visual references. Replacing expected results after task creation invalidates the task. After the agent exits, the runner rehashes the original bundle, copies it into a per-trial private staging directory, and rehashes both original and staged trees before and after grading.
+
+“Hidden” is an evaluation-disclosure property: grader bytes are absent from the prompt and candidate workspace. Without an OS filesystem sandbox it is not a secrecy guarantee against a malicious process that probes arbitrary host paths; this limitation is recorded in TaskRun.
+
+Supported checks are:
+
+- `data-pack-contract`: `SGDataLoader` errors and optional strict warnings;
+- `library-rules`: trusted library rules, optionally selected by rule id;
+- `extract-equivalence`: configured literal/fromPack equivalence and complete coverage;
+- `command`: shell-free argv with bounded cwd, timeout and output;
+- `runtime-evidence` and `visual-evidence`: strict evidence consumers.
+
+Every check declares `id`, `type`, positive `weight`, and whether it is `required`. Results distinguish `passed`, `failed`, `not-assessed`, `invalid`, and `infra-error`. A required non-passed check fails the verdict; `not-assessed` never earns score. Candidate command timeouts, malformed evidence, candidate browser failures, and candidate extraction worker exits are failures that remain in the AI denominator. `infra-error` is reserved for preflight/dependency failures such as a missing browser or explicitly declared provider/grader infrastructure exit code. If a trial observes both candidate failure and infra error, candidate failure takes precedence for aggregation. Command evidence includes argv, exit/signal/timeout, duration, bounded stdout/stderr and their SHA-256.
+
+The grader receives `{workspace}`, `{grader}`, `{artifacts}`, `{toolRoot}`, `{treeSha256}`, and `{taskId}` token substitutions. `{grader}` and `{toolRoot}` refer to per-trial verified staging copies. It cannot change the accepted patch: candidate tree digest is fixed before grading and post-grader tree comparison rejects any candidate mutation. `extract-equivalence` uses a trusted supervisor process plus a worker-thread subject: the result travels over a private transferred `MessagePort` with a random token, candidate stdout/stderr are not result channels, and the trusted wrapper must return exactly one authenticated result with exit 0. Timeout, `process.exit`, forged stdout markers, or malformed worker output cannot terminate the parent TaskRun or earn a pass.
diff --git a/references/patch-execution-contract.md b/references/patch-execution-contract.md
new file mode 100644
index 0000000..13a0187
--- /dev/null
+++ b/references/patch-execution-contract.md
@@ -0,0 +1,29 @@
+# Patch execution and PatchAudit v1.0
+
+`sg-data-pack task run` never asks an agent to modify the source library. It copies the exact
+content-addressed source tree into a disposable workspace and gives the agent one output capability:
+a unified diff artifact.
+
+The runner then:
+
+1. validates the task, input, source tree, grader spec and grader tree digests;
+2. snapshots source and workspace trees;
+3. runs the agent with a bounded shell-free argv command;
+4. rejects direct source/workspace writes and undeclared artifact files;
+5. parses the diff and rejects absolute/traversal paths, binary patches, renames, mode changes, symlinks, submodules, duplicate entries and unsupported operations; add/delete patches must declare modes, v1 adds are fixed to `100644`, delete mode must match the baseline, and postflight independently rejects actual mode drift;
+6. enforces `allowed`/`forbidden` paths before apply, with deny precedence;
+7. runs exact `git apply --check` and `git apply` with no fuzz;
+8. snapshots the candidate and rechecks the actual tree diff against file policy;
+9. after agent execution, rehashes the task, source, grader, external integrity inputs, and tool tree;
+10. copies graders and tool scripts into per-trial digest-verified private staging directories, then rehashes original and staged trees before/after grading;
+11. runs candidate extraction/equivalence code in a bounded child worker rather than the parent runner;
+12. fails closed if the final source tree differs or any post-agent immutable binding drifts;
+13. writes a SHA-256-bound `patch-audit.json` and TaskRun artifacts.
+
+`PatchAudit` records task/patch/tree digests, declared and actual operations, policy decisions,
+apply command evidence and capability limits. Any rejected patch leaves the disposable workspace at
+its original tree or the whole workspace is discarded.
+
+## Trust boundary
+
+The v1 runner provides strong postcondition and content-integrity enforcement. It does **not** claim a portable OS sandbox: `osSandbox`, `networkIsolation`, and `processIsolation` are recorded as false unless a future platform runner can enforce them. A local process can address host paths outside its cwd; `allowedRoot` bounds command cwd, not filesystem syscalls. Therefore private staging and post-stage rehashing detect drift but do not make grader bytes secret from a malicious local agent. An external AI provider may need network access even when candidate code execution is intended to be offline. Capability absence must remain visible in TaskRun and must not be reworded as isolation.
diff --git a/references/run-report-contract.md b/references/run-report-contract.md
index d6e91f7..e43f658 100644
--- a/references/run-report-contract.md
+++ b/references/run-report-contract.md
@@ -36,7 +36,7 @@ The default terminal and `REPORT.md` output answer these questions in order:
4. **剩余风险** — warnings, unassessed capabilities, derivation impacts, no-check rules, and known library boundaries;
5. **下一步** — prioritized owner, command, and done-when condition.
-`NOT_ASSESSED` is never treated as a pass. Runtime DOM mount, visual regression, and live production crawl remain `not-assessed` unless a future evidence collector is connected.
+`NOT_ASSESSED` is never treated as a pass. Library Evolution Report v1 does not execute browsers, so runtime DOM mount, visual regression, and live production crawl remain `not-assessed` here. Agent TaskRun has separate RuntimeEvidence/VisualEvidence collectors; those task artifacts are not silently projected into this older report contract.
## Outcomes and exit codes
diff --git a/references/runtime-visual-evidence-contract.md b/references/runtime-visual-evidence-contract.md
new file mode 100644
index 0000000..4e94868
--- /dev/null
+++ b/references/runtime-visual-evidence-contract.md
@@ -0,0 +1,37 @@
+# Runtime and visual evidence contract v1.0
+
+Runtime and visual checks are evidence producers, not booleans copied from an external exit code.
+`sg-data-pack` validates the evidence shape, binds every local artifact by SHA-256, and recomputes
+threshold results. Missing measurements or missing artifacts produce `not-assessed`, never `passed`.
+
+## RuntimeEvidence
+
+A `runtime-evidence` object binds `subjectTreeSha256`, producer name/version, environment and one
+scenario. `assertions` contain stable ids and `passed|failed`; console errors, page errors and network
+failures are independent failure sources. At least one assertion is required for an assessed result.
+Artifacts use paths relative to the evidence file and include `sha256:<64 lowercase hex>`. Paths are mandatory; duplicate ids/paths, symlink or hardlink components, missing files, and digest mismatches make the evidence non-passing. Runtime evidence requires at least one verified artifact and one assertion for an assessed result.
+
+`evidenceId` is `runtime:` over canonical evidence with `evidenceId` omitted.
+
+## VisualEvidence
+
+A `visual-evidence` object binds reference/candidate tree digests, visual configuration digest,
+producer and thresholds. Each scenario contains a viewport, measured pixel-diff ratio, computed-style
+score, coverage, stability failures and three artifact references: reference, candidate and diff.
+
+The consumer recomputes:
+
+- `pixelDiffRatio <= maxPixelDiffRatio`;
+- `computedStyleScore >= minComputedStyleScore`;
+- `coverage >= minCoverage`;
+- `stabilityFailures === 0`;
+- all three distinct screenshot/diff references resolve to verified image artifacts;
+- all artifact paths exist, contain no symlink/hardlink escape, and match their SHA-256.
+
+The benchmark browser producer preflights browser, Pillow, reference render, and reference self-comparison before subject execution; those dependency failures are infrastructure errors. Candidate render/hang/comparison failures exit as candidate failures. Coverage comes from rendered card count, stability comes from a second candidate screenshot, and console/page/network failures come from page instrumentation rather than hardcoded empty arrays.
+
+`evidenceId` is `visual:` over canonical evidence with `evidenceId` omitted. A producer's own
+`passed` field, if present, is not authoritative.
+
+The canonical ui-dismantler quality profile remains documented in `visual-regression.md`. This
+contract is its auditable handoff format; it does not make sg-data-pack a browser implementation.
diff --git a/references/task-run-contract.md b/references/task-run-contract.md
new file mode 100644
index 0000000..26b036b
--- /dev/null
+++ b/references/task-run-contract.md
@@ -0,0 +1,24 @@
+# Agent TaskRun v1.0
+
+`TaskRun` is the auditable result of one AgentTaskManifest execution. It is separate from Library
+Evolution `RunReport v1.0`: the latter explains a Data Pack state; TaskRun records an agent,
+patch-policy decision, graders and runtime/visual evidence.
+
+A TaskRun binds:
+
+- task manifest bytes, taskId, revision and source contract tree digest;
+- independent runner snapshots before/final and `sourceUnchanged`;
+- complete source/candidate snapshots including `.git/**`, before/after untrusted stages and at finalization;
+- digest-verified private staging copies of grader and tool trees plus original/staged integrity checks;
+- any post-agent task/grader/tool/candidate/source integrity drift as a non-passing policy violation;
+- provider/model metadata, argv, duration, exit/signal/timeout, stdout/stderr digests;
+- agent patch bytes and PatchAudit;
+- candidate tree digest and every GradeReport;
+- every prompt, patch, audit, DOM, screenshot, diff and evidence artifact by SHA-256;
+- `secondaryErrors` for post-grade integrity/artifact problems that must not erase an already observed candidate failure;
+- explicit capabilities and missing isolation guarantees.
+
+Verdicts are `passed`, `agent-failed`, `patch-invalid`, `policy-violation`, `grader-failed`, `input-error`, or `infra-error`. Provider authentication/unavailability and trusted dependency preflight failures are infrastructure failures. Candidate/subject timeout, malformed evidence, browser failure after subject execution, and post-agent integrity drift are completed non-passes and remain in the AI denominator. `termination.processExitCode` uses 0 for pass, 1 for completed non-pass, 2 for invalid input, and 3 for infrastructure failure.
+
+TaskRun reports can prove what happened in one disposable workspace. They do not authorize promotion
+of the patch into the source library; promotion is a separate reviewed operation.
diff --git a/research/agent-eval/PROMPT-TO-ARTIFACT.md b/research/agent-eval/PROMPT-TO-ARTIFACT.md
new file mode 100644
index 0000000..970b22b
--- /dev/null
+++ b/research/agent-eval/PROMPT-TO-ARTIFACT.md
@@ -0,0 +1,40 @@
+# Prompt-to-artifact completion audit
+
+This checklist maps the requested next-stage capability to concrete implementation and executed evidence.
+
+| Requirement | Implementation | Executed evidence | Status |
+|---|---|---|---|
+| Agent task manifest | `scripts/lib/agent-task.schema.json`, `scripts/lib/agent-task-manifest.js`, `scripts/sg-pack-task.js` | Three valid content-addressed manifests under `tasks/*/task.json`; full source/input/grader/tree verification in `audit-results.js` | Complete |
+| AI patch execution | `scripts/lib/sg-task-runner.js`, `scripts/lib/sg-patch-executor.js` | 15 real-AI TaskRuns under `results/claude/trials/`; each keeps prompt, patch where produced, PatchAudit, source before/final digests and termination | Complete |
+| Task-specific grader | `scripts/lib/sg-grader.js`, `scripts/lib/sg-grader.schema.json` | Complete hidden expected candidates for all three tasks; Pattern C GradeReport has 5 checks and score 8/8 in `results/claude/trials/pattern-c-runtime-r04/grade-task-grader.json` | Complete |
+| Allowed/forbidden enforcement | Manifest file policy, parser preflight, exact apply, postflight tree diff, mode gate | `results/policy-negative/patch-audit.json` rejects `.git/config`; tests reject direct `.git` mutation, traversal, binary, rename, hardlink/symlink, executable add and actual mode drift | Complete |
+| Runtime evidence | `scripts/lib/sg-runtime-evidence.js`; staged headless-Chrome producer | `results/claude/trials/pattern-c-runtime-r04/runtime-evidence.json`, verified DOM/log artifacts, 4 passed assertions, zero console/page/network failures | Complete for one real-AI trial; corpus rate 1/4 valid required trials |
+| Visual evidence | `scripts/lib/sg-visual-evidence.js`; Chrome + Pillow producer | `results/claude/trials/pattern-c-runtime-r04/visual-evidence.json`, reference/candidate/diff screenshots, second-run stability, pixel/style/coverage gates | Complete for one real-AI trial; corpus rate 1/4 valid required trials |
+| Success-rate experiment | `scripts/lib/sg-experiment.js`, `scripts/sg-pack-experiment.js`, `experiment.schema.json` | Scripted 15/15 control and Claude 15-trial report under `results/`; Wilson CI, stage denominators, provider model/tokens/cost and failure taxonomy | Complete |
+| Auditable | SHA-256 content IDs; verified grader/tool staging; immutable-input checks; worker supervisor; raw/relocated reports | `audit-results.js` verifies 30 TaskRuns, 188 artifacts, all content IDs, relocation audits, evidence, negative control and executed tooling tree | Complete |
+| Regressible | Contract, security, adversarial and integration tests | Final `node --test tests/*.test.js`: 188/188 pass; `git diff --check`: pass | Complete |
+| No overstated claim | Capability flags and contracts explicitly deny unavailable isolation; strict claim levels | Scripted: `harness-only`; real AI: `ai-static-contract-only`; runtime/visual subset 1/4, not represented as fully demonstrated | Complete |
+
+## Real AI conclusion
+
+The final real-provider experiment measured 15 trials: 14 valid, 1 infrastructure timeout, 7 passes and 7 valid patch failures. Valid pass rate was 50.0% with Wilson 95% interval 26.8%–73.2%. The actual provider-reported model was `MiniMax-M3[1M]` despite the declared `sonnet` alias.
+
+This is initial, limited evidence that an AI can use the contracts for the benchmark's static component-library data tasks. It is **not** evidence that the runtime/visual Pattern C task is reliably solved: only 1 of 4 valid required trials passed runtime and visual gates. It is also not evidence of arbitrary production-library autonomy.
+
+## Explicit limitations
+
+- No OS sandbox, filesystem syscall isolation, network isolation, process isolation, or malicious-agent grader secrecy is claimed.
+- “Hidden grader” means absent from prompt and candidate workspace. Per-trial verified staging and rehashing detect drift but do not make host paths unreadable to an unsandboxed malicious process.
+- The benchmark is three small synthetic tasks, not a production-library sample.
+- Provider timeout is reported as infrastructure and was not retried.
+- Exact patch application rejects malformed hunk context; seven valid AI trials failed at this stage.
+
+## Reproduction
+
+```bash
+node --test tests/*.test.js
+node research/agent-eval/audit-results.js
+node research/agent-eval/run-policy-negative.js /tmp/sg-policy-negative
+```
+
+The exact tooling used for the saved experiments is preserved under `results/executed-tooling/` and bound by `execution-bundle.json`. Raw execution reports and deterministic relocation audits are retained next to the self-contained ExperimentReports.
diff --git a/research/agent-eval/README.md b/research/agent-eval/README.md
new file mode 100644
index 0000000..3620687
--- /dev/null
+++ b/research/agent-eval/README.md
@@ -0,0 +1,75 @@
+# Agent task evaluation benchmark
+
+This directory contains the reproducible synthetic benchmark used to test whether an agent can make a correct, auditable component-library change under the sg-data-pack contracts.
+
+## Corpus
+
+The frozen corpus contains three tasks, each repeated five times by the experiment specs:
+
+1. `field-update`: change one existing entity field and preserve every unrelated value;
+2. `alias-relation`: add one alias and one registered relation without changing entities;
+3. `pattern-c-runtime`: extend a Pattern C DOM JSON gallery, compile the browser data entry, and pass real runtime and visual checks.
+
+Each task binds source/input bytes, full source tree including modes, allow/deny file policy, patch limits, complete grader directory, required evidence scenarios, and execution limits through `taskId`. Complete expected candidates and browser references are in the grader bundle, not in the agent prompt or candidate workspace.
+
+“Hidden” does not mean secret from a malicious local process on an unsandboxed host. The portable runner records `osSandbox`, `filesystemIsolation`, `networkIsolation`, `processIsolation`, and `maliciousAgentGraderSecrecy` as false. It instead rehashes immutable inputs and runs graders/tools from verified per-trial staging copies after agent execution.
+
+## Rebuild frozen identities
+
+After intentionally changing a task, source fixture, grader, reference, provider adapter, or experiment definition:
+
+```bash
+node research/agent-eval/build-fixtures.js
+node scripts/sg-data-pack task validate research/agent-eval/tasks/field-update/task.json
+node scripts/sg-data-pack task validate research/agent-eval/tasks/alias-relation/task.json
+node scripts/sg-data-pack task validate research/agent-eval/tasks/pattern-c-runtime/task.json
+```
+
+Never regenerate IDs merely to hide unexplained drift. Review the changed bytes first.
+
+## Run the harness control
+
+```bash
+rm -rf /tmp/sg-agent-scripted
+node scripts/sg-data-pack experiment \
+ research/agent-eval/experiment-scripted.json \
+ --out /tmp/sg-agent-scripted
+```
+
+The scripted provider is a deterministic positive control. Its only valid claim level is `harness-only`; 100% scripted success is not AI evidence.
+
+## Run a real provider
+
+```bash
+rm -rf /tmp/sg-agent-claude
+node scripts/sg-data-pack experiment \
+ research/agent-eval/experiment-claude.json \
+ --out /tmp/sg-agent-claude
+```
+
+The provider adapter bytes are content-bound by the ExperimentSpec and the actual provider/model/token/cost metadata is retained when reported by the CLI. Authentication, provider unavailability, and trusted dependency preflight failures are reported as infrastructure errors. Candidate patch, policy, grader, timeout, runtime, visual, or post-agent integrity failures remain in the valid AI denominator.
+
+## Interpret results
+
+Always report:
+
+- total, valid, invalid, and infrastructure trial counts;
+- valid pass rate and Wilson 95% interval;
+- agent completion, patch apply, policy, grader, runtime, and visual stage counts;
+- actual provider-reported model names and usage/cost;
+- failure taxonomy and every TaskRun artifact path;
+- the explicit `claimLevel` and missing sandbox capabilities.
+
+`ai-task-contract-with-runtime-visual-subset` is evidence only for this small synthetic task corpus. It is not evidence of arbitrary production-library autonomy. `ai-static-contract-only` means at least one AI trial passed but the declared browser subset did not pass completely. `ai-task-contract-not-demonstrated` means no valid AI trial passed.
+
+## Audit artifacts
+
+An experiment output is self-contained below its output directory:
+
+- `experiment.json` and `EXPERIMENT.md`;
+- `trials//task-run.json`;
+- agent prompt and patch;
+- PatchAudit and GradeReport;
+- runtime/visual evidence, DOM, browser logs, screenshots, and diffs where required.
+
+Recompute every listed artifact SHA-256 before publishing a result. Do not replace missing real-provider evidence with scripted or mocked output.
diff --git a/research/agent-eval/audit-results.js b/research/agent-eval/audit-results.js
new file mode 100644
index 0000000..fb6c0ec
--- /dev/null
+++ b/research/agent-eval/audit-results.js
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const ROOT = path.resolve(__dirname, '../..');
+const { contentId, sha256Bytes, stableJson } = require(path.join(ROOT, 'scripts/lib/sg-evidence-utils.js'));
+const { validateExperimentSpec, aggregate } = require(path.join(ROOT, 'scripts/lib/sg-experiment.js'));
+const { validateTaskManifest } = require(path.join(ROOT, 'scripts/lib/agent-task-manifest.js'));
+const { readRuntimeEvidence } = require(path.join(ROOT, 'scripts/lib/sg-runtime-evidence.js'));
+const { readVisualEvidence } = require(path.join(ROOT, 'scripts/lib/sg-visual-evidence.js'));
+const { snapshotTree } = require(path.join(ROOT, 'scripts/lib/sg-tree-snapshot.js'));
+
+function assert(condition, message) { if (!condition) throw new Error(message); }
+function equal(left, right, message) { assert(stableJson(left) === stableJson(right), message); }
+
+for (const task of ['field-update', 'alias-relation', 'pattern-c-runtime']) {
+ const file = path.join(__dirname, 'tasks', task, 'task.json');
+ const manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
+ const result = validateTaskManifest(manifest, { taskFile: file, verifyFiles: true, verifyTree: true });
+ assert(result.valid, `${task} manifest: ${JSON.stringify(result.issues)}`);
+}
+
+let taskRuns = 0;
+let artifacts = 0;
+for (const name of ['scripted', 'claude']) {
+ const resultRoot = path.join(__dirname, 'results', name);
+ const reportFile = path.join(resultRoot, 'experiment.json');
+ const rawFile = path.join(resultRoot, 'experiment.raw.json');
+ const report = JSON.parse(fs.readFileSync(reportFile, 'utf8'));
+ const raw = JSON.parse(fs.readFileSync(rawFile, 'utf8'));
+ const specFile = path.join(__dirname, name === 'scripted' ? 'experiment-scripted.json' : 'experiment-claude.json');
+ const specBytes = fs.readFileSync(specFile);
+ const spec = JSON.parse(specBytes);
+ const validation = validateExperimentSpec(spec);
+ assert(validation.valid, `${name} spec: ${validation.errors.join('; ')}`);
+ assert(report.resultId === contentId('experiment-result', report, ['resultId']), `${name} relocated resultId`);
+ assert(raw.resultId === contentId('experiment-result', raw, ['resultId']), `${name} raw resultId`);
+ assert(report.specSha256 === sha256Bytes(specBytes), `${name} spec digest`);
+ assert(report.taskSetSha256 === sha256Bytes(stableJson(spec.tasks)), `${name} task-set digest`);
+ equal(report.summary, aggregate(report.trials, spec.agent.kind), `${name} aggregate`);
+ equal(report.summary, raw.summary, `${name} summary changed during relocation`);
+ const audit = JSON.parse(fs.readFileSync(path.join(resultRoot, 'relocation-audit.json'), 'utf8'));
+ assert(audit.auditId === contentId('relocation-audit', audit, ['auditId']), `${name} relocation auditId`);
+ assert(audit.raw.resultId === raw.resultId && audit.relocated.resultId === report.resultId, `${name} relocation ids`);
+ assert(audit.raw.sha256 === sha256Bytes(fs.readFileSync(rawFile)), `${name} raw relocation digest`);
+ assert(audit.relocated.sha256 === sha256Bytes(fs.readFileSync(reportFile)), `${name} relocated digest`);
+ for (const trial of report.trials) {
+ if (!trial.runId) continue;
+ assert(trial.artifactPath === `trials/${trial.trialId}/task-run.json`, `${trial.trialId} non-portable artifactPath`);
+ const runFile = path.join(resultRoot, trial.artifactPath);
+ const run = JSON.parse(fs.readFileSync(runFile, 'utf8'));
+ taskRuns += 1;
+ assert(run.runId === trial.runId && run.runId === contentId('task-run', run, ['runId']), `${trial.trialId} runId`);
+ assert(run.task.sourceUnchanged, `${trial.trialId} source drift`);
+ for (const artifact of run.artifacts) {
+ const file = path.resolve(path.dirname(runFile), artifact.path);
+ const relative = path.relative(path.dirname(runFile), file);
+ assert(relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative), `${trial.trialId} artifact escape`);
+ assert(sha256Bytes(fs.readFileSync(file)) === artifact.sha256, `${trial.trialId} artifact ${artifact.path}`);
+ artifacts += 1;
+ }
+ if (run.patch) assert(run.patch.auditId === contentId('patch-audit', run.patch, ['auditId']), `${trial.trialId} patch auditId`);
+ for (const grade of run.grades) assert(grade.gradeId === contentId('grade', grade, ['gradeId']), `${trial.trialId} gradeId`);
+ const runtimeFile = path.join(path.dirname(runFile), 'runtime-evidence.json');
+ if (fs.existsSync(runtimeFile)) assert(readRuntimeEvidence(runtimeFile).validation.status === 'passed', `${trial.trialId} runtime evidence`);
+ const visualFile = path.join(path.dirname(runFile), 'visual-evidence.json');
+ if (fs.existsSync(visualFile)) assert(readVisualEvidence(visualFile).validation.status === 'passed', `${trial.trialId} visual evidence`);
+ }
+ console.log(`${name}: ${report.summary.passed}/${report.summary.validTrials} valid passes; ${report.summary.claimLevel}`);
+}
+
+const policy = JSON.parse(fs.readFileSync(path.join(__dirname, 'results', 'policy-negative', 'result.json'), 'utf8'));
+assert(policy.verdict === 'policy-violation' && policy.sourceUnchanged, 'policy negative control');
+const tooling = JSON.parse(fs.readFileSync(path.join(__dirname, 'results', 'executed-tooling', 'execution-bundle.json'), 'utf8'));
+const toolingTree = 'sha256:' + snapshotTree(path.join(__dirname, 'results', 'executed-tooling', 'scripts'), { errorOnSpecialFile: true, exclude: false }).treeSha256;
+assert(tooling.scriptsTreeSha256 === toolingTree, 'executed tooling tree digest');
+console.log(`verified ${taskRuns} TaskRuns, ${artifacts} artifacts, relocation audits, evidence, policy control, and executed tooling`);
diff --git a/research/agent-eval/build-fixtures.js b/research/agent-eval/build-fixtures.js
new file mode 100644
index 0000000..33c361d
--- /dev/null
+++ b/research/agent-eval/build-fixtures.js
@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const ROOT = path.resolve(__dirname, '../..');
+const { computeTaskId, computeTreeSha256, sha256 } = require(path.join(ROOT, 'scripts/lib/agent-task-manifest.js'));
+const { contentId, sha256Bytes } = require(path.join(ROOT, 'scripts/lib/sg-evidence-utils.js'));
+
+const tasksRoot = path.join(__dirname, 'tasks');
+const taskDefinitions = [
+ {
+ id: 'field-update',
+ instructions: 'In lib/data/data.json, change only entities.alice.summary from summary-old to exactly summary-new. Preserve the valid Data Pack shape and every unrelated value. Emit a unified diff only.',
+ inputs: ['lib/data/data.json'],
+ allowed: [{ pattern: 'lib/data/data.json', operations: ['modify'] }],
+ grader: 'grader/spec.json',
+ evidence: { runtime: [], visual: [] },
+ },
+ {
+ id: 'alias-relation',
+ instructions: 'In lib/data/data.json, add alias Alicia that resolves to existing entity alice. Also add exactly one master relation with id alice-bob-friend, a alice, b bob, type friend, and label Friend. Do not create entities or modify existing names. Emit a unified diff only.',
+ inputs: ['lib/data/data.json'],
+ allowed: [{ pattern: 'lib/data/data.json', operations: ['modify'] }],
+ grader: 'grader/spec.json',
+ evidence: { runtime: [], visual: [] },
+ },
+ {
+ id: 'pattern-c-runtime',
+ instructions: 'Extend the Pattern C card gallery with a third card. In lib/data/data.json add entity gamma with kind card, title Gamma Card, and color #8b5cf6; add alias Gamma Card -> gamma; append gamma to the gallery stage entities and domain.cardOrder. Update lib/data/data.js so SG_DATA_PACK exactly mirrors data.json. Do not modify the engine or HTML. Emit a unified diff only.',
+ inputs: ['lib/data/data.json', 'lib/data/data.js', 'lib/src/cards.js', 'lib/examples/index.html'],
+ allowed: [
+ { pattern: 'lib/data/data.json', operations: ['modify'] },
+ { pattern: 'lib/data/data.js', operations: ['modify'] },
+ ],
+ grader: 'grader/spec.json',
+ evidence: { runtime: ['pattern-c-gallery'], visual: ['pattern-c-gallery-desktop'] },
+ },
+];
+
+for (const definition of taskDefinitions) {
+ const directory = path.join(tasksRoot, definition.id);
+ const source = path.join(directory, 'source');
+ const instructions = { text: definition.instructions, sha256: sha256(definition.instructions) };
+ const graderFile = path.join(directory, definition.grader);
+ const graderDir = path.dirname(graderFile);
+ const manifest = {
+ taskVersion: '1.0', taskId: null, instructions,
+ source: { root: 'source', revision: `${definition.id}-fixture-v1`, treeSha256: computeTreeSha256(source) },
+ inputs: definition.inputs.map((relative, index) => ({ id: `input-${index + 1}`, path: relative, role: index === 0 ? 'canonical-data' : 'integration-input', sha256: sha256(fs.readFileSync(path.join(source, relative))) })),
+ filePolicy: {
+ allowed: definition.allowed,
+ forbidden: ['.git/**', 'lib/src/**', 'lib/examples/**'],
+ denyPrecedence: true, allowSymlinks: false, allowHardlinks: false,
+ maxChangedFiles: definition.allowed.length, maxPatchBytes: 262144,
+ },
+ patchPolicy: { format: 'unified-diff', fuzz: 0, allowBinary: false },
+ graders: [{ id: 'task-grader', spec: definition.grader, sha256: sha256(fs.readFileSync(graderFile)), treeSha256: computeTreeSha256(graderDir), weight: 1 }],
+ evidence: definition.evidence,
+ execution: { timeoutMs: 480000, network: 'off', maxOutputBytes: 4194304 },
+ };
+ manifest.taskId = computeTaskId(manifest);
+ fs.writeFileSync(path.join(directory, 'task.json'), JSON.stringify(manifest, null, 2) + '\n');
+}
+
+function experiment(kind, provider, model, executable) {
+ const spec = {
+ experimentVersion: '1.0', experimentId: null,
+ title: `${provider} agent task benchmark`,
+ agent: {
+ kind, provider, model,
+ argv: ['{node}', '{adapter}', '{workspace}', '{patch}', '{prompt}'],
+ adapter: { path: `providers/${executable}`, sha256: sha256Bytes(fs.readFileSync(path.join(__dirname, 'providers', executable))) },
+ ...(kind === 'ai' ? { infraExitCodes: [3] } : {}),
+ },
+ tasks: taskDefinitions.map((task) => {
+ const relative = `tasks/${task.id}/task.json`;
+ const file = path.join(__dirname, relative);
+ const bytes = fs.readFileSync(file);
+ return { id: task.id, manifest: relative, taskId: JSON.parse(bytes).taskId, manifestSha256: sha256Bytes(bytes) };
+ }),
+ repetitions: 5,
+ seed: 'fixed-task-order-v1',
+ };
+ experimentId(spec);
+ return spec;
+}
+function experimentId(spec) {
+ const material = { ...spec }; delete material.experimentId;
+ spec.experimentId = contentId('experiment', material, []);
+}
+fs.writeFileSync(path.join(__dirname, 'experiment-scripted.json'), JSON.stringify(experiment('scripted', 'deterministic-fixture', 'scripted-v1', 'scripted-patch-agent.js'), null, 2) + '\n');
+fs.writeFileSync(path.join(__dirname, 'experiment-codex.json'), JSON.stringify(experiment('ai', 'codex-cli-0.139.0', 'gpt-5.5 (provider-reported)', 'codex-patch-agent.js'), null, 2) + '\n');
+fs.writeFileSync(path.join(__dirname, 'experiment-claude.json'), JSON.stringify(experiment('ai', 'claude-code-2.1.185', 'sonnet alias (provider metadata recorded)', 'claude-patch-agent.js'), null, 2) + '\n');
+console.log('Agent evaluation fixtures regenerated.');
diff --git a/research/agent-eval/experiment-claude.json b/research/agent-eval/experiment-claude.json
new file mode 100644
index 0000000..66e29f6
--- /dev/null
+++ b/research/agent-eval/experiment-claude.json
@@ -0,0 +1,46 @@
+{
+ "experimentVersion": "1.0",
+ "experimentId": "experiment:20c81f58da37a1b7f17b7239e57dd9d7036ebce72529827e387f7ba48d536de3",
+ "title": "claude-code-2.1.185 agent task benchmark",
+ "agent": {
+ "kind": "ai",
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "{node}",
+ "{adapter}",
+ "{workspace}",
+ "{patch}",
+ "{prompt}"
+ ],
+ "adapter": {
+ "path": "providers/claude-patch-agent.js",
+ "sha256": "sha256:81bb58c7df706063d66686fdd2d888a1e3143e57e60d631ace40bc28218a27e0"
+ },
+ "infraExitCodes": [
+ 3
+ ]
+ },
+ "tasks": [
+ {
+ "id": "field-update",
+ "manifest": "tasks/field-update/task.json",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "manifestSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a"
+ },
+ {
+ "id": "alias-relation",
+ "manifest": "tasks/alias-relation/task.json",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "manifestSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630"
+ },
+ {
+ "id": "pattern-c-runtime",
+ "manifest": "tasks/pattern-c-runtime/task.json",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "manifestSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7"
+ }
+ ],
+ "repetitions": 5,
+ "seed": "fixed-task-order-v1"
+}
diff --git a/research/agent-eval/experiment-codex.json b/research/agent-eval/experiment-codex.json
new file mode 100644
index 0000000..9473091
--- /dev/null
+++ b/research/agent-eval/experiment-codex.json
@@ -0,0 +1,46 @@
+{
+ "experimentVersion": "1.0",
+ "experimentId": "experiment:352bdabeddb1a9ce24430e6c482249da01a4c22e0bdc7bf66c36d6449a399f62",
+ "title": "codex-cli-0.139.0 agent task benchmark",
+ "agent": {
+ "kind": "ai",
+ "provider": "codex-cli-0.139.0",
+ "model": "gpt-5.5 (provider-reported)",
+ "argv": [
+ "{node}",
+ "{adapter}",
+ "{workspace}",
+ "{patch}",
+ "{prompt}"
+ ],
+ "adapter": {
+ "path": "providers/codex-patch-agent.js",
+ "sha256": "sha256:03539068e04f8e82cf0757a670e8e799e8860aaa9f6abe7a912df2b8ef12559e"
+ },
+ "infraExitCodes": [
+ 3
+ ]
+ },
+ "tasks": [
+ {
+ "id": "field-update",
+ "manifest": "tasks/field-update/task.json",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "manifestSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a"
+ },
+ {
+ "id": "alias-relation",
+ "manifest": "tasks/alias-relation/task.json",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "manifestSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630"
+ },
+ {
+ "id": "pattern-c-runtime",
+ "manifest": "tasks/pattern-c-runtime/task.json",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "manifestSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7"
+ }
+ ],
+ "repetitions": 5,
+ "seed": "fixed-task-order-v1"
+}
diff --git a/research/agent-eval/experiment-scripted.json b/research/agent-eval/experiment-scripted.json
new file mode 100644
index 0000000..cf2139b
--- /dev/null
+++ b/research/agent-eval/experiment-scripted.json
@@ -0,0 +1,43 @@
+{
+ "experimentVersion": "1.0",
+ "experimentId": "experiment:c914c326de4a45b64ab26744732c39e2d77885280e35269d009e233bc560cf3b",
+ "title": "deterministic-fixture agent task benchmark",
+ "agent": {
+ "kind": "scripted",
+ "provider": "deterministic-fixture",
+ "model": "scripted-v1",
+ "argv": [
+ "{node}",
+ "{adapter}",
+ "{workspace}",
+ "{patch}",
+ "{prompt}"
+ ],
+ "adapter": {
+ "path": "providers/scripted-patch-agent.js",
+ "sha256": "sha256:b67eeeb225c65f2e3f1aa55f1cc0818b9efece0c428a316b40a776157c5fd7b2"
+ }
+ },
+ "tasks": [
+ {
+ "id": "field-update",
+ "manifest": "tasks/field-update/task.json",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "manifestSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a"
+ },
+ {
+ "id": "alias-relation",
+ "manifest": "tasks/alias-relation/task.json",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "manifestSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630"
+ },
+ {
+ "id": "pattern-c-runtime",
+ "manifest": "tasks/pattern-c-runtime/task.json",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "manifestSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7"
+ }
+ ],
+ "repetitions": 5,
+ "seed": "fixed-task-order-v1"
+}
diff --git a/research/agent-eval/providers/claude-patch-agent.js b/research/agent-eval/providers/claude-patch-agent.js
new file mode 100644
index 0000000..001bee7
--- /dev/null
+++ b/research/agent-eval/providers/claude-patch-agent.js
@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const [, , workspace, patchFile, promptFile] = process.argv;
+if (!workspace || !patchFile || !promptFile) {
+ console.error('Usage: claude-patch-agent ');
+ process.exit(2);
+}
+
+function collectTextFiles(root) {
+ const files = [];
+ let bytes = 0;
+ function visit(directory) {
+ for (const name of fs.readdirSync(directory).sort()) {
+ if (name === '.git') continue;
+ const absolute = path.join(directory, name);
+ const stat = fs.lstatSync(absolute);
+ if (stat.isSymbolicLink() || !stat.isFile() && !stat.isDirectory()) throw new Error(`unsupported source entry: ${absolute}`);
+ if (stat.isDirectory()) visit(absolute);
+ else {
+ const content = fs.readFileSync(absolute);
+ if (content.includes(0)) throw new Error(`binary source is not supported by this provider: ${absolute}`);
+ bytes += content.length;
+ if (bytes > 1024 * 1024) throw new Error('source context exceeds 1 MiB');
+ files.push({ path: path.relative(root, absolute).split(path.sep).join('/'), content: content.toString('utf8') });
+ }
+ }
+ }
+ visit(root);
+ return files;
+}
+
+try {
+ const task = fs.readFileSync(promptFile, 'utf8');
+ const files = collectTextFiles(workspace);
+ const source = files.map((file) => `--- SOURCE FILE: ${file.path} ---\n${file.content}\n--- END SOURCE FILE ---`).join('\n\n');
+ const prompt = `${task}\n\nYou are participating in an audited patch-only evaluation. The complete source snapshot is included below. You have no tools and must not request any. Produce one exact unified diff against these source bytes. Use diff --git, --- a/path, +++ b/path headers and zero-fuzz context. Preserve all unrelated bytes and values. Do not use Markdown fences, commentary, prose, commands, binary patches, renames, symlinks, or mode changes. Your response must begin with diff --git and contain only the patch.\n\n${source}`;
+ const result = spawnSync('claude', [
+ '-p', '--safe-mode', '--tools', '', '--no-session-persistence',
+ '--model', 'sonnet', '--output-format', 'json', prompt,
+ ], { encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: 300000 });
+ if (result.error) throw result.error;
+ if (result.status !== 0) {
+ if (result.stderr) process.stderr.write(result.stderr);
+ process.exit(3);
+ }
+ const response = JSON.parse(result.stdout);
+ if (response.is_error || response.subtype !== 'success' || typeof response.result !== 'string') {
+ console.error(response.api_error_status || response.subtype || 'Claude provider failed');
+ process.exit(3);
+ }
+ let patch = response.result.trim();
+ const start = patch.indexOf('diff --git ');
+ if (start > 0) patch = patch.slice(start);
+ if (patch.startsWith('```')) patch = patch.replace(/^```(?:diff)?\s*/, '').replace(/\s*```$/, '');
+ fs.writeFileSync(patchFile, patch.trimEnd() + '\n');
+ const metadata = {
+ provider: 'claude-code',
+ declaredModel: 'sonnet',
+ modelUsage: response.modelUsage || {},
+ usage: response.usage || {},
+ totalCostUsd: response.total_cost_usd === undefined ? null : response.total_cost_usd,
+ durationApiMs: response.duration_api_ms || null,
+ stopReason: response.stop_reason || null,
+ permissionDenials: response.permission_denials || [],
+ };
+ process.stdout.write(JSON.stringify(metadata) + '\n');
+} catch (error) {
+ console.error(error.stack || error.message);
+ process.exit(3);
+}
diff --git a/research/agent-eval/providers/codex-patch-agent.js b/research/agent-eval/providers/codex-patch-agent.js
new file mode 100644
index 0000000..afc5854
--- /dev/null
+++ b/research/agent-eval/providers/codex-patch-agent.js
@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const { spawnSync } = require('node:child_process');
+
+const [, , workspace, patchFile, promptFile] = process.argv;
+if (!workspace || !patchFile || !promptFile) {
+ console.error('Usage: codex-patch-agent ');
+ process.exit(2);
+}
+const task = fs.readFileSync(promptFile, 'utf8');
+const prompt = `${task}\n\nYou are participating in an audited patch-only evaluation. Inspect the workspace, but do not modify it. Produce one exact unified diff against the current files. Use diff --git, --- a/path, +++ b/path headers and zero-fuzz context. Do not use Markdown fences, commentary, prose, commands, binary patches, renames, symlinks, or mode changes. The final response must begin with diff --git and contain only the patch.`;
+const result = spawnSync('codex', [
+ 'exec', '--ephemeral', '--ignore-user-config', '--ignore-rules',
+ '--sandbox', 'read-only', '--skip-git-repo-check', '-C', workspace,
+ '--color', 'never', '-o', patchFile, '-'
+], { input: prompt, encoding: 'utf8', maxBuffer: 4 * 1024 * 1024, timeout: 300000 });
+if (result.stdout) process.stdout.write(result.stdout);
+if (result.stderr) process.stderr.write(result.stderr);
+if (result.error) {
+ console.error(result.error.message);
+ process.exit(3);
+}
+if (result.status !== 0) process.exit(3);
+let patch = fs.readFileSync(patchFile, 'utf8').trim();
+const start = patch.indexOf('diff --git ');
+if (start > 0) patch = patch.slice(start);
+if (patch.startsWith('```')) patch = patch.replace(/^```(?:diff)?\s*/, '').replace(/\s*```$/, '');
+fs.writeFileSync(patchFile, patch.trimEnd() + '\n');
diff --git a/research/agent-eval/providers/scripted-patch-agent.js b/research/agent-eval/providers/scripted-patch-agent.js
new file mode 100644
index 0000000..76b0022
--- /dev/null
+++ b/research/agent-eval/providers/scripted-patch-agent.js
@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+
+const [, , workspace, patchFile, promptFile] = process.argv;
+const prompt = fs.readFileSync(promptFile, 'utf8');
+const changes = [];
+function updateJson(relative, mutate) {
+ const file = path.join(workspace, relative);
+ const before = fs.readFileSync(file, 'utf8');
+ const value = JSON.parse(before);
+ mutate(value);
+ changes.push({ relative, before, after: JSON.stringify(value, null, 2) + '\n' });
+}
+if (prompt.includes('summary-new')) {
+ updateJson('lib/data/data.json', (pack) => { pack.entities.alice.summary = 'summary-new'; });
+} else if (prompt.includes('Alicia') && prompt.includes('friend')) {
+ updateJson('lib/data/data.json', (pack) => {
+ pack.aliases.Alicia = 'alice';
+ pack.relations.push({ id: 'alice-bob-friend', a: 'alice', b: 'bob', type: 'friend', label: 'Friend' });
+ });
+} else if (prompt.includes('Gamma Card')) {
+ updateJson('lib/data/data.json', (pack) => {
+ pack.entities.gamma = { kind: 'card', title: 'Gamma Card', color: '#8b5cf6' };
+ pack.aliases['Gamma Card'] = 'gamma';
+ pack.stages.find((stage) => stage.key === 'gallery').entities.push('gamma');
+ pack.domain.cardOrder.push('gamma');
+ });
+ const data = JSON.parse(changes[0].after);
+ const relative = 'lib/data/data.js';
+ changes.push({ relative, before: fs.readFileSync(path.join(workspace, relative), 'utf8'), after: 'globalThis.SG_DATA_PACK = ' + JSON.stringify(data) + ';\n' });
+} else {
+ console.error('scripted agent does not recognize this task');
+ process.exit(1);
+}
+function fullFileDiff(change) {
+ const before = change.before.trimEnd().split('\n');
+ const after = change.after.trimEnd().split('\n');
+ return [
+ `diff --git a/${change.relative} b/${change.relative}`,
+ `--- a/${change.relative}`,
+ `+++ b/${change.relative}`,
+ `@@ -1,${before.length} +1,${after.length} @@`,
+ ...before.map((line) => '-' + line),
+ ...after.map((line) => '+' + line),
+ ].join('\n');
+}
+fs.writeFileSync(patchFile, changes.map(fullFileDiff).join('\n') + '\n');
diff --git a/research/agent-eval/relocate-results.js b/research/agent-eval/relocate-results.js
new file mode 100644
index 0000000..42abc69
--- /dev/null
+++ b/research/agent-eval/relocate-results.js
@@ -0,0 +1,56 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const ROOT = path.resolve(__dirname, '../..');
+const { contentId, sha256Bytes } = require(path.join(ROOT, 'scripts/lib/sg-evidence-utils.js'));
+const { renderExperiment } = require(path.join(ROOT, 'scripts/lib/sg-experiment.js'));
+
+const directory = path.resolve(process.argv[2] || '');
+if (!process.argv[2] || !fs.existsSync(path.join(directory, 'experiment.json'))) {
+ console.error('Usage: relocate-results ');
+ process.exit(2);
+}
+const reportFile = path.join(directory, 'experiment.json');
+const rawFile = path.join(directory, 'experiment.raw.json');
+const rawMarkdown = path.join(directory, 'EXPERIMENT.raw.md');
+const auditFile = path.join(directory, 'relocation-audit.json');
+if (fs.existsSync(rawFile) || fs.existsSync(rawMarkdown) || fs.existsSync(auditFile)) {
+ console.error('Relocation outputs already exist; refusing to overwrite: ' + directory);
+ process.exit(2);
+}
+const rawBytes = fs.readFileSync(reportFile);
+const rawReport = JSON.parse(rawBytes);
+if (rawReport.resultId !== contentId('experiment-result', rawReport, ['resultId'])) throw new Error('raw ExperimentReport resultId is invalid');
+const relocated = JSON.parse(rawBytes);
+const transformations = [];
+for (const trial of relocated.trials) {
+ if (!trial.runId) continue;
+ const expected = `trials/${trial.trialId}/task-run.json`;
+ const runFile = path.join(directory, expected);
+ if (!fs.existsSync(runFile)) throw new Error(`missing TaskRun for ${trial.trialId}: ${runFile}`);
+ const run = JSON.parse(fs.readFileSync(runFile, 'utf8'));
+ if (run.runId !== trial.runId) throw new Error(`TaskRun id mismatch for ${trial.trialId}`);
+ if (trial.artifactPath !== expected) transformations.push({ trialId: trial.trialId, from: trial.artifactPath, to: expected });
+ trial.artifactPath = expected;
+}
+if (!transformations.length) throw new Error('ExperimentReport is already self-contained; no relocation needed');
+relocated.resultId = contentId('experiment-result', relocated, ['resultId']);
+const relocatedBytes = Buffer.from(JSON.stringify(relocated, null, 2) + '\n');
+const audit = {
+ auditVersion: '1.0', auditId: null, kind: 'experiment-relocation',
+ reason: 'Canonicalize trial TaskRun references after copying an experiment produced through a symlinked output-root spelling.',
+ experimentId: relocated.experimentId,
+ raw: { resultId: rawReport.resultId, sha256: sha256Bytes(rawBytes), file: 'experiment.raw.json' },
+ relocated: { resultId: relocated.resultId, sha256: sha256Bytes(relocatedBytes), file: 'experiment.json' },
+ transformations,
+ semanticFieldsUnchanged: ['experimentId', 'specSha256', 'taskSetSha256', 'agent', 'seed', 'repetitions', 'summary', 'trial verdicts', 'runIds'],
+};
+audit.auditId = contentId('relocation-audit', audit, ['auditId']);
+fs.renameSync(reportFile, rawFile);
+const markdownFile = path.join(directory, 'EXPERIMENT.md');
+if (fs.existsSync(markdownFile)) fs.renameSync(markdownFile, rawMarkdown);
+fs.writeFileSync(reportFile, relocatedBytes);
+fs.writeFileSync(markdownFile, renderExperiment(relocated));
+fs.writeFileSync(auditFile, JSON.stringify(audit, null, 2) + '\n');
+console.log(`Relocated ${transformations.length} TaskRun references: ${relocated.resultId}`);
diff --git a/research/agent-eval/results/README.md b/research/agent-eval/results/README.md
new file mode 100644
index 0000000..5fc29d8
--- /dev/null
+++ b/research/agent-eval/results/README.md
@@ -0,0 +1,65 @@
+# Final agent evaluation results
+
+These are the complete final audit bundles produced after the hardened runner, grader, evidence, denominator, and integrity reviews. They are committed as evidence, not as generated examples.
+
+## Deterministic harness control
+
+- Directory: [`scripted/`](scripted/)
+- Experiment: `experiment:c914c326de4a45b64ab26744732c39e2d77885280e35269d009e233bc560cf3b`
+- Self-contained result: `experiment-result:118e0cb1259834a153aa0bf69c0818810a5dbdc240806b167224e1c399cda2f2`
+- Raw execution result: `experiment-result:ef44e721234db91d12ca3154dab1daa9156e463195ada83d30e757cedff296e8`
+- Spec digest: `sha256:d7bf7564fa4283c928ffe7930b4cbaeac2d91f6ca14f994f5ebd1533967d62c2`
+- Trials: 15 total, 15 valid, 0 infrastructure errors
+- Passed: 15/15
+- Runtime subset: 5/5
+- Visual subset: 5/5
+- Wilson 95% interval: 79.6%–100.0%
+- Claim level: `harness-only`
+- Recomputed audit: 15 TaskRuns and 125 listed artifact digests
+
+This control proves only that the harness accepts deterministic correct patches and produces complete evidence. It is not AI capability evidence.
+
+## Real AI experiment
+
+- Directory: [`claude/`](claude/)
+- Experiment: `experiment:20c81f58da37a1b7f17b7239e57dd9d7036ebce72529827e387f7ba48d536de3`
+- Self-contained result: `experiment-result:43c9a914a08639bd2adbe306d81cddef9dfe863ae8eaa5205c7e3cf22c8e1581`
+- Raw execution result: `experiment-result:56bbca6f13b5ea63f767f6ef4edcb2588959dfa59cb409e66c26981af559de91`
+- Spec digest: `sha256:5dc9c394f2375420f7bf15f2cf4bd9a5e60e6db78887a3f01e3f2a7782486629`
+- Declared provider: `claude-code-2.1.185`, `sonnet` alias
+- Provider-reported actual model: `MiniMax-M3[1M]`
+- Trials: 15 total, 14 valid, 1 infrastructure error
+- Passed: 7/14 valid trials (50.0%)
+- Wilson 95% interval: 26.8%–73.2%
+- Failure taxonomy: 7 passed, 7 patch-invalid, 1 provider timeout infrastructure error
+- Runtime subset: 1/4 valid required trials
+- Visual subset: 1/4 valid required trials
+- Provider usage: 22,223 input tokens; 94,361 output tokens; 1,792 cache-read input tokens
+- Provider-reported cost: $2.471036
+- Claim level: `ai-static-contract-only`
+- Recomputed audit: 15 TaskRuns and 63 listed artifact digests
+
+The four valid Pattern C trials stay in the runtime/visual denominator even when a patch fails before evidence collection. One passed the complete Data Pack, hidden candidate, browser producer, runtime, and visual gates. Three produced patches that failed exact zero-fuzz application. A fifth Pattern C trial hit the frozen provider timeout and is reported separately as infrastructure rather than retried.
+
+The measured AI therefore showed initial, limited success on the static task contract, but the complete runtime/visual component-modification subset was **not** reliably demonstrated. This result does not support a claim of arbitrary production-library autonomy.
+
+## Integrity and limitations
+
+Every saved TaskRun records source immutability, patch policy, grader results, capabilities, and relative artifact digests. The audit recomputed every listed artifact SHA-256 after copying these bundles into the repository.
+
+The portable runner does not provide an OS filesystem sandbox, network isolation, process isolation, or malicious-agent grader secrecy. Hidden grader bytes are absent from the prompt and candidate workspace, then copied into per-trial digest-verified staging after agent execution; original and staged bindings are rechecked. These controls detect drift but do not make arbitrary host paths unreadable to a malicious local process.
+
+## Revalidation
+
+```bash
+node --test tests/*.test.js
+node scripts/sg-data-pack task validate research/agent-eval/tasks/field-update/task.json
+node scripts/sg-data-pack task validate research/agent-eval/tasks/alias-relation/task.json
+node scripts/sg-data-pack task validate research/agent-eval/tasks/pattern-c-runtime/task.json
+node scripts/sg-data-pack evidence runtime \
+ research/agent-eval/results/claude/trials/pattern-c-runtime-r04/runtime-evidence.json
+node scripts/sg-data-pack evidence visual \
+ research/agent-eval/results/claude/trials/pattern-c-runtime-r04/visual-evidence.json
+```
+
+See each directory's `experiment.json`, `EXPERIMENT.md`, and `trials/*/task-run.json` for the complete machine-readable evidence.
diff --git a/research/agent-eval/results/claude/EXPERIMENT.md b/research/agent-eval/results/claude/EXPERIMENT.md
new file mode 100644
index 0000000..15906b7
--- /dev/null
+++ b/research/agent-eval/results/claude/EXPERIMENT.md
@@ -0,0 +1,34 @@
+# Agent success-rate experiment
+
+- Experiment: `experiment:20c81f58da37a1b7f17b7239e57dd9d7036ebce72529827e387f7ba48d536de3`
+- Agent: claude-code-2.1.185 / sonnet alias (provider metadata recorded) (ai)
+- Trials: 15; valid 14; infra 1
+- Pass rate: **50.0%** (Wilson 95% 26.8%–73.2%)
+- Claim level: **ai-static-contract-only**
+- Actual model(s): MiniMax-M3[1M]
+- Tokens: input 22223; output 94361; cache read 1792
+- Recorded provider cost: $2.471036
+
+## Failure taxonomy
+
+- infra-error: 1
+- passed: 7
+- patch-invalid: 7
+
+## Trials
+
+- field-update-r01: patch-invalid; patch=rejected; grades=none
+- field-update-r02: patch-invalid; patch=rejected; grades=none
+- field-update-r03: passed; patch=applied; grades=passed
+- field-update-r04: passed; patch=applied; grades=passed
+- field-update-r05: passed; patch=applied; grades=passed
+- alias-relation-r01: passed; patch=applied; grades=passed
+- alias-relation-r02: passed; patch=applied; grades=passed
+- alias-relation-r03: patch-invalid; patch=rejected; grades=none
+- alias-relation-r04: patch-invalid; patch=rejected; grades=none
+- alias-relation-r05: passed; patch=applied; grades=passed
+- pattern-c-runtime-r01: patch-invalid; patch=rejected; grades=none
+- pattern-c-runtime-r02: infra-error; patch=none; grades=none
+- pattern-c-runtime-r03: patch-invalid; patch=rejected; grades=none
+- pattern-c-runtime-r04: passed; patch=applied; grades=passed
+- pattern-c-runtime-r05: patch-invalid; patch=rejected; grades=none
diff --git a/research/agent-eval/results/claude/EXPERIMENT.raw.md b/research/agent-eval/results/claude/EXPERIMENT.raw.md
new file mode 100644
index 0000000..15906b7
--- /dev/null
+++ b/research/agent-eval/results/claude/EXPERIMENT.raw.md
@@ -0,0 +1,34 @@
+# Agent success-rate experiment
+
+- Experiment: `experiment:20c81f58da37a1b7f17b7239e57dd9d7036ebce72529827e387f7ba48d536de3`
+- Agent: claude-code-2.1.185 / sonnet alias (provider metadata recorded) (ai)
+- Trials: 15; valid 14; infra 1
+- Pass rate: **50.0%** (Wilson 95% 26.8%–73.2%)
+- Claim level: **ai-static-contract-only**
+- Actual model(s): MiniMax-M3[1M]
+- Tokens: input 22223; output 94361; cache read 1792
+- Recorded provider cost: $2.471036
+
+## Failure taxonomy
+
+- infra-error: 1
+- passed: 7
+- patch-invalid: 7
+
+## Trials
+
+- field-update-r01: patch-invalid; patch=rejected; grades=none
+- field-update-r02: patch-invalid; patch=rejected; grades=none
+- field-update-r03: passed; patch=applied; grades=passed
+- field-update-r04: passed; patch=applied; grades=passed
+- field-update-r05: passed; patch=applied; grades=passed
+- alias-relation-r01: passed; patch=applied; grades=passed
+- alias-relation-r02: passed; patch=applied; grades=passed
+- alias-relation-r03: patch-invalid; patch=rejected; grades=none
+- alias-relation-r04: patch-invalid; patch=rejected; grades=none
+- alias-relation-r05: passed; patch=applied; grades=passed
+- pattern-c-runtime-r01: patch-invalid; patch=rejected; grades=none
+- pattern-c-runtime-r02: infra-error; patch=none; grades=none
+- pattern-c-runtime-r03: patch-invalid; patch=rejected; grades=none
+- pattern-c-runtime-r04: passed; patch=applied; grades=passed
+- pattern-c-runtime-r05: patch-invalid; patch=rejected; grades=none
diff --git a/research/agent-eval/results/claude/experiment.json b/research/agent-eval/results/claude/experiment.json
new file mode 100644
index 0000000..a58545a
--- /dev/null
+++ b/research/agent-eval/results/claude/experiment.json
@@ -0,0 +1,866 @@
+{
+ "reportVersion": "1.0",
+ "resultId": "experiment-result:43c9a914a08639bd2adbe306d81cddef9dfe863ae8eaa5205c7e3cf22c8e1581",
+ "experimentId": "experiment:20c81f58da37a1b7f17b7239e57dd9d7036ebce72529827e387f7ba48d536de3",
+ "specSha256": "sha256:5dc9c394f2375420f7bf15f2cf4bd9a5e60e6db78887a3f01e3f2a7782486629",
+ "seed": "fixed-task-order-v1",
+ "agent": {
+ "kind": "ai",
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)"
+ },
+ "taskSetSha256": "sha256:f47d3569a2df49c15aa0d82c8125ea2e4c1efcce3d7de4d2b6333f6967ca6ea7",
+ "repetitions": 5,
+ "summary": {
+ "totalTrials": 15,
+ "validTrials": 14,
+ "invalidTrials": 0,
+ "infraErrors": 1,
+ "passed": 7,
+ "passRate": 0.5,
+ "confidence95": {
+ "lower": 0.26799202452413634,
+ "upper": 0.7320079754758637
+ },
+ "verdicts": {
+ "infra-error": 1,
+ "passed": 7,
+ "patch-invalid": 7
+ },
+ "stages": {
+ "agentCompleted": {
+ "passed": 14,
+ "total": 14
+ },
+ "patchApplied": {
+ "passed": 7,
+ "total": 14
+ },
+ "policyCompliant": {
+ "passed": 7,
+ "total": 14
+ },
+ "graderPassed": {
+ "passed": 7,
+ "total": 14
+ },
+ "runtimePassed": {
+ "passed": 1,
+ "total": 4
+ },
+ "visualPassed": {
+ "passed": 1,
+ "total": 4
+ }
+ },
+ "provider": {
+ "actualModels": [
+ "MiniMax-M3[1M]"
+ ],
+ "usage": {
+ "inputTokens": 22223,
+ "outputTokens": 94361,
+ "cacheReadInputTokens": 1792,
+ "totalCostUsd": 2.4710360000000002
+ }
+ },
+ "durationMs": {
+ "mean": 74086.59953592857,
+ "median": 54848.1158125
+ },
+ "claimLevel": "ai-static-contract-only"
+ },
+ "trials": [
+ {
+ "trialId": "field-update-r01",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:eda458f8bd0740b373639a702e700e17e55e8fb73e2dda8c5b0b187a3de0fe7c",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 135,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.009463999999999998,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 135,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.009463999999999998,
+ "durationApiMs": 2651,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 3909.933917,
+ "artifactPath": "trials/field-update-r01/task-run.json"
+ },
+ {
+ "trialId": "field-update-r02",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:309918a0fb60925f85b679ec333015c77b33eec76d0030b7dcdb9624b5ea525b",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 167,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.010263999999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 167,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.010263999999999999,
+ "durationApiMs": 2453,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 2826.63475,
+ "artifactPath": "trials/field-update-r02/task-run.json"
+ },
+ {
+ "trialId": "field-update-r03",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:6ebe4b699f124bdb070f4ec2b44f8a2533dcc0b2951ced11677219ab27cb3233",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1204,
+ "outputTokens": 1163,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.035159,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1204,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 1163,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.035159,
+ "durationApiMs": 20145,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 20638.498834,
+ "artifactPath": "trials/field-update-r03/task-run.json"
+ },
+ {
+ "trialId": "field-update-r04",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:73c5544c5dc6f6c78c4320e2888567420ae338d827a2a39e9b86f17b1c4c2f33",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1204,
+ "outputTokens": 162,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.010133999999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1204,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 162,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.010133999999999999,
+ "durationApiMs": 2291,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 2758.782,
+ "artifactPath": "trials/field-update-r04/task-run.json"
+ },
+ {
+ "trialId": "field-update-r05",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:528e87fb2aa72f2ccadb88f732444149a30535c80fc1eff8692c2f6c251a9873",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 2639,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.07206399999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 2639,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.07206399999999999,
+ "durationApiMs": 34735,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 35221.021209,
+ "artifactPath": "trials/field-update-r05/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r01",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:4c5cfca83f5d0e68b86925e1b1181f829d019ef54b4e752caf20582406f68f7f",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1286,
+ "outputTokens": 9456,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.24289400000000003,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1286,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9456,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.24289400000000003,
+ "durationApiMs": 64470,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 65048.181125,
+ "artifactPath": "trials/alias-relation-r01/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r02",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:5752bc64b30328c65d022760d19df872a2d7d570418d6ae3b5edf0ed31b41608",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1284,
+ "outputTokens": 6789,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.176209,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1284,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 6789,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.176209,
+ "durationApiMs": 43993,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 44648.0505,
+ "artifactPath": "trials/alias-relation-r02/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r03",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:f198854b1a316d4f255f970c87ed2763e7938159828a18d060c647a0abcfecad",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1285,
+ "outputTokens": 454,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.017839,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1285,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 454,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.017839,
+ "durationApiMs": 8821,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 9842.414625,
+ "artifactPath": "trials/alias-relation-r03/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r04",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:6c4a7a66037f3e6a9cf0029744d4fc930e1cb95a7da420acd8e1a6552b622d09",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1286,
+ "outputTokens": 5806,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.151644,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1286,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 5806,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.151644,
+ "durationApiMs": 72771,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 73767.052167,
+ "artifactPath": "trials/alias-relation-r04/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r05",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:9c06d2f4dcd88ef9c0894e318e07d4e61331b53315b665487b9bb4a441fa921b",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1285,
+ "outputTokens": 9192,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.236289,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1285,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9192,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.236289,
+ "durationApiMs": 105475,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 105986.162542,
+ "artifactPath": "trials/alias-relation-r05/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r01",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:d7c57685e7c52ac0c0cf5b5479eacc353a887873f09b066aea87e26baf9c5495",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2443,
+ "outputTokens": 19966,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.511429,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2443,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 19966,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.511429,
+ "durationApiMs": 231215,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 231612.691334,
+ "artifactPath": "trials/pattern-c-runtime-r01/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r02",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:1e758add780932a28d4a8805f1027723d18ea577ceef2b37205920c827061b04",
+ "verdict": "infra-error",
+ "agentExitCode": 3,
+ "patchStatus": null,
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": null,
+ "durationMs": 300087.777416,
+ "artifactPath": "trials/pattern-c-runtime-r02/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r03",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:495af23afb9d35b3d74f0547b57ef22f80e85b980877159bc46e3e6db566f3f1",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2444,
+ "outputTokens": 9733,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.25560900000000003,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2444,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9733,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.25560900000000003,
+ "durationApiMs": 100259,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 100649.2315,
+ "artifactPath": "trials/pattern-c-runtime-r03/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r04",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:d760c86dff6d4d960aca95fe2473ce10baaa537bc94534635abc413a1aab1310",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [
+ "passed"
+ ],
+ "visualStatuses": [
+ "passed"
+ ],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2443,
+ "outputTokens": 11854,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.308629,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2443,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 11854,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.308629,
+ "durationApiMs": 126661,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 140822.51575,
+ "artifactPath": "trials/pattern-c-runtime-r04/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r05",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:07e9c45767c9629be80216750400fa6df478397e9d3bca19e06fe327616100a7",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2444,
+ "outputTokens": 16845,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.433409,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2444,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 16845,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.433409,
+ "durationApiMs": 198830,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 199481.22325,
+ "artifactPath": "trials/pattern-c-runtime-r05/task-run.json"
+ }
+ ]
+}
diff --git a/research/agent-eval/results/claude/experiment.raw.json b/research/agent-eval/results/claude/experiment.raw.json
new file mode 100644
index 0000000..8f4122b
--- /dev/null
+++ b/research/agent-eval/results/claude/experiment.raw.json
@@ -0,0 +1,866 @@
+{
+ "reportVersion": "1.0",
+ "resultId": "experiment-result:56bbca6f13b5ea63f767f6ef4edcb2588959dfa59cb409e66c26981af559de91",
+ "experimentId": "experiment:20c81f58da37a1b7f17b7239e57dd9d7036ebce72529827e387f7ba48d536de3",
+ "specSha256": "sha256:5dc9c394f2375420f7bf15f2cf4bd9a5e60e6db78887a3f01e3f2a7782486629",
+ "seed": "fixed-task-order-v1",
+ "agent": {
+ "kind": "ai",
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)"
+ },
+ "taskSetSha256": "sha256:f47d3569a2df49c15aa0d82c8125ea2e4c1efcce3d7de4d2b6333f6967ca6ea7",
+ "repetitions": 5,
+ "summary": {
+ "totalTrials": 15,
+ "validTrials": 14,
+ "invalidTrials": 0,
+ "infraErrors": 1,
+ "passed": 7,
+ "passRate": 0.5,
+ "confidence95": {
+ "lower": 0.26799202452413634,
+ "upper": 0.7320079754758637
+ },
+ "verdicts": {
+ "infra-error": 1,
+ "passed": 7,
+ "patch-invalid": 7
+ },
+ "stages": {
+ "agentCompleted": {
+ "passed": 14,
+ "total": 14
+ },
+ "patchApplied": {
+ "passed": 7,
+ "total": 14
+ },
+ "policyCompliant": {
+ "passed": 7,
+ "total": 14
+ },
+ "graderPassed": {
+ "passed": 7,
+ "total": 14
+ },
+ "runtimePassed": {
+ "passed": 1,
+ "total": 4
+ },
+ "visualPassed": {
+ "passed": 1,
+ "total": 4
+ }
+ },
+ "provider": {
+ "actualModels": [
+ "MiniMax-M3[1M]"
+ ],
+ "usage": {
+ "inputTokens": 22223,
+ "outputTokens": 94361,
+ "cacheReadInputTokens": 1792,
+ "totalCostUsd": 2.4710360000000002
+ }
+ },
+ "durationMs": {
+ "mean": 74086.59953592857,
+ "median": 54848.1158125
+ },
+ "claimLevel": "ai-static-contract-only"
+ },
+ "trials": [
+ {
+ "trialId": "field-update-r01",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:eda458f8bd0740b373639a702e700e17e55e8fb73e2dda8c5b0b187a3de0fe7c",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 135,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.009463999999999998,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 135,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.009463999999999998,
+ "durationApiMs": 2651,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 3909.933917,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r01/task-run.json"
+ },
+ {
+ "trialId": "field-update-r02",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:309918a0fb60925f85b679ec333015c77b33eec76d0030b7dcdb9624b5ea525b",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 167,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.010263999999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 167,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.010263999999999999,
+ "durationApiMs": 2453,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 2826.63475,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r02/task-run.json"
+ },
+ {
+ "trialId": "field-update-r03",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:6ebe4b699f124bdb070f4ec2b44f8a2533dcc0b2951ced11677219ab27cb3233",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1204,
+ "outputTokens": 1163,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.035159,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1204,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 1163,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.035159,
+ "durationApiMs": 20145,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 20638.498834,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r03/task-run.json"
+ },
+ {
+ "trialId": "field-update-r04",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:73c5544c5dc6f6c78c4320e2888567420ae338d827a2a39e9b86f17b1c4c2f33",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1204,
+ "outputTokens": 162,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.010133999999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1204,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 162,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.010133999999999999,
+ "durationApiMs": 2291,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 2758.782,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r04/task-run.json"
+ },
+ {
+ "trialId": "field-update-r05",
+ "taskSlug": "field-update",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "runId": "task-run:528e87fb2aa72f2ccadb88f732444149a30535c80fc1eff8692c2f6c251a9873",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 2639,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.07206399999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 2639,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.07206399999999999,
+ "durationApiMs": 34735,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 35221.021209,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r05/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r01",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:4c5cfca83f5d0e68b86925e1b1181f829d019ef54b4e752caf20582406f68f7f",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1286,
+ "outputTokens": 9456,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.24289400000000003,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1286,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9456,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.24289400000000003,
+ "durationApiMs": 64470,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 65048.181125,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r01/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r02",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:5752bc64b30328c65d022760d19df872a2d7d570418d6ae3b5edf0ed31b41608",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1284,
+ "outputTokens": 6789,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.176209,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1284,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 6789,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.176209,
+ "durationApiMs": 43993,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 44648.0505,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r02/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r03",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:f198854b1a316d4f255f970c87ed2763e7938159828a18d060c647a0abcfecad",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1285,
+ "outputTokens": 454,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.017839,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1285,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 454,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.017839,
+ "durationApiMs": 8821,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 9842.414625,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r03/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r04",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:6c4a7a66037f3e6a9cf0029744d4fc930e1cb95a7da420acd8e1a6552b622d09",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1286,
+ "outputTokens": 5806,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.151644,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1286,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 5806,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.151644,
+ "durationApiMs": 72771,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 73767.052167,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r04/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r05",
+ "taskSlug": "alias-relation",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "runId": "task-run:9c06d2f4dcd88ef9c0894e318e07d4e61331b53315b665487b9bb4a441fa921b",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 0,
+ "visualRequired": 0,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1285,
+ "outputTokens": 9192,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.236289,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1285,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9192,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.236289,
+ "durationApiMs": 105475,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 105986.162542,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r05/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r01",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:d7c57685e7c52ac0c0cf5b5479eacc353a887873f09b066aea87e26baf9c5495",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2443,
+ "outputTokens": 19966,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.511429,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2443,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 19966,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.511429,
+ "durationApiMs": 231215,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 231612.691334,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r01/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r02",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:1e758add780932a28d4a8805f1027723d18ea577ceef2b37205920c827061b04",
+ "verdict": "infra-error",
+ "agentExitCode": 3,
+ "patchStatus": null,
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": null,
+ "durationMs": 300087.777416,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r02/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r03",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:495af23afb9d35b3d74f0547b57ef22f80e85b980877159bc46e3e6db566f3f1",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2444,
+ "outputTokens": 9733,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.25560900000000003,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2444,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9733,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.25560900000000003,
+ "durationApiMs": 100259,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 100649.2315,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r03/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r04",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:d760c86dff6d4d960aca95fe2473ce10baaa537bc94534635abc413a1aab1310",
+ "verdict": "passed",
+ "agentExitCode": 0,
+ "patchStatus": "applied",
+ "gradeVerdicts": [
+ "passed"
+ ],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [
+ "passed"
+ ],
+ "visualStatuses": [
+ "passed"
+ ],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2443,
+ "outputTokens": 11854,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.308629,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2443,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 11854,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.308629,
+ "durationApiMs": 126661,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 140822.51575,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r04/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r05",
+ "taskSlug": "pattern-c-runtime",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "runId": "task-run:07e9c45767c9629be80216750400fa6df478397e9d3bca19e06fe327616100a7",
+ "verdict": "patch-invalid",
+ "agentExitCode": 0,
+ "patchStatus": "rejected",
+ "gradeVerdicts": [],
+ "runtimeRequired": 1,
+ "visualRequired": 1,
+ "runtimeStatuses": [],
+ "visualStatuses": [],
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2444,
+ "outputTokens": 16845,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.433409,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2444,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 16845,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.433409,
+ "durationApiMs": 198830,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ },
+ "durationMs": 199481.22325,
+ "artifactPath": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r05/task-run.json"
+ }
+ ]
+}
diff --git a/research/agent-eval/results/claude/relocation-audit.json b/research/agent-eval/results/claude/relocation-audit.json
new file mode 100644
index 0000000..4b0be3b
--- /dev/null
+++ b/research/agent-eval/results/claude/relocation-audit.json
@@ -0,0 +1,105 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "relocation-audit:15adcf52d9ccaa7fc9fa97407a2436e931891a56029cadf2aaa8e299d3721731",
+ "kind": "experiment-relocation",
+ "reason": "Canonicalize trial TaskRun references after copying an experiment produced through a symlinked output-root spelling.",
+ "experimentId": "experiment:20c81f58da37a1b7f17b7239e57dd9d7036ebce72529827e387f7ba48d536de3",
+ "raw": {
+ "resultId": "experiment-result:56bbca6f13b5ea63f767f6ef4edcb2588959dfa59cb409e66c26981af559de91",
+ "sha256": "sha256:cdc25ee69647fa93529b4d1b02de72cc912dbf5953461fe78ea16563766e74c4",
+ "file": "experiment.raw.json"
+ },
+ "relocated": {
+ "resultId": "experiment-result:43c9a914a08639bd2adbe306d81cddef9dfe863ae8eaa5205c7e3cf22c8e1581",
+ "sha256": "sha256:c1b84ad911e99ae38c1961e88711a64c26c0f2bb76423f3a21f9e80bae83a3e8",
+ "file": "experiment.json"
+ },
+ "transformations": [
+ {
+ "trialId": "field-update-r01",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r01/task-run.json",
+ "to": "trials/field-update-r01/task-run.json"
+ },
+ {
+ "trialId": "field-update-r02",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r02/task-run.json",
+ "to": "trials/field-update-r02/task-run.json"
+ },
+ {
+ "trialId": "field-update-r03",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r03/task-run.json",
+ "to": "trials/field-update-r03/task-run.json"
+ },
+ {
+ "trialId": "field-update-r04",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r04/task-run.json",
+ "to": "trials/field-update-r04/task-run.json"
+ },
+ {
+ "trialId": "field-update-r05",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/field-update-r05/task-run.json",
+ "to": "trials/field-update-r05/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r01",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r01/task-run.json",
+ "to": "trials/alias-relation-r01/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r02",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r02/task-run.json",
+ "to": "trials/alias-relation-r02/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r03",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r03/task-run.json",
+ "to": "trials/alias-relation-r03/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r04",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r04/task-run.json",
+ "to": "trials/alias-relation-r04/task-run.json"
+ },
+ {
+ "trialId": "alias-relation-r05",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/alias-relation-r05/task-run.json",
+ "to": "trials/alias-relation-r05/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r01",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r01/task-run.json",
+ "to": "trials/pattern-c-runtime-r01/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r02",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r02/task-run.json",
+ "to": "trials/pattern-c-runtime-r02/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r03",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r03/task-run.json",
+ "to": "trials/pattern-c-runtime-r03/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r04",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r04/task-run.json",
+ "to": "trials/pattern-c-runtime-r04/task-run.json"
+ },
+ {
+ "trialId": "pattern-c-runtime-r05",
+ "from": "../../private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r05/task-run.json",
+ "to": "trials/pattern-c-runtime-r05/task-run.json"
+ }
+ ],
+ "semanticFieldsUnchanged": [
+ "experimentId",
+ "specSha256",
+ "taskSetSha256",
+ "agent",
+ "seed",
+ "repetitions",
+ "summary",
+ "trial verdicts",
+ "runIds"
+ ]
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r01/TASK.md b/research/agent-eval/results/claude/trials/alias-relation-r01/TASK.md
new file mode 100644
index 0000000..f4381c1
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r01/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, add alias Alicia that resolves to existing entity alice. Also add exactly one master relation with id alice-bob-friend, a alice, b bob, type friend, and label Friend. Do not create entities or modify existing names. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r01/agent.patch b/research/agent-eval/results/claude/trials/alias-relation-r01/agent.patch
new file mode 100644
index 0000000..4cf438b
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r01/agent.patch
@@ -0,0 +1,26 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -16,6 +16,7 @@
+ },
+ "aliases": {
+ "Alice": "alice",
++ "Alicia": "alice",
+ "Bob": "bob"
+ },
+ "relationTypes": {
+@@ -23,5 +23,13 @@
+ "label": "Friend"
+ }
+ },
+- "relations": [],
++ "relations": [
++ {
++ "id": "alice-bob-friend",
++ "a": "alice",
++ "b": "bob",
++ "type": "friend",
++ "label": "Friend"
++ }
++ ],
+ "stages": [
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r01/grade-task-grader.json b/research/agent-eval/results/claude/trials/alias-relation-r01/grade-task-grader.json
new file mode 100644
index 0000000..27a3753
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r01/grade-task-grader.json
@@ -0,0 +1,65 @@
+{
+ "gradeVersion": "1.0",
+ "gradeId": "grade:48758757945eece3abe919926dea1c4a6c86124970b81753fd524d3ec8645061",
+ "graderId": "agent-alias-relation-grader",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "treeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "alias-relation"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 30.828667,
+ "stdoutSha256": "sha256:4a3966d3a5a5052d2f32c8dc8a3aed53e4834383504caa9459c2442b66a0aded",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "alias-relation: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r01/patch-audit.json b/research/agent-eval/results/claude/trials/alias-relation-r01/patch-audit.json
new file mode 100644
index 0000000..7e6d24b
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r01/patch-audit.json
@@ -0,0 +1,102 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:9f2b9556ce9916bba43f00d4ee4cd64e18faca5f88dace4e5bccc9cffb65411d",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "applied",
+ "patchSha256": "sha256:2045a256902a6b7cd61330dd30a15767cc32bdf4f0fdea243ed7fe4a97c1f517",
+ "patchBytes": 489,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 2,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "0dd120528e6773e0dfc3a76bf4740870ba17fd32da59aae1b1abf7895ef676db",
+ "size": 586,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "41b5ca0d459b453de00b5d66ef8578e23bffad35792a069ee460eebd4282366e",
+ "size": 742,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573969869-70a183d32ae7d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 23.527167,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573969869-70a183d32ae7d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 18.968208,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r01/task-run.json b/research/agent-eval/results/claude/trials/alias-relation-r01/task-run.json
new file mode 100644
index 0000000..8930b88
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r01/task-run.json
@@ -0,0 +1,298 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:4c5cfca83f5d0e68b86925e1b1181f829d019ef54b4e752caf20582406f68f7f",
+ "task": {
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "taskFileSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630",
+ "sourceRevision": "alias-relation-fixture-v1",
+ "contractTreeSha256": "sha256:fffbe5c6cd3b100ea754dfc73b72bb75adae0ccd8ca789fb18c00ed68b2cfc53",
+ "runnerSnapshotBeforeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "runnerSnapshotFinalSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "alias-relation-r01"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r01/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r01/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 64885.879875,
+ "stdoutSha256": "sha256:5f065e3e6ebd7e789adfd88e524d6352de434248d48d26960cfee6b73644b221",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1286,\"outputTokens\":9456,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.24289400000000003,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1286,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":9456,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.24289400000000003,\"durationApiMs\":64470,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1286,
+ "outputTokens": 9456,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.24289400000000003,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1286,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9456,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.24289400000000003,
+ "durationApiMs": 64470,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:9f2b9556ce9916bba43f00d4ee4cd64e18faca5f88dace4e5bccc9cffb65411d",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "applied",
+ "patchSha256": "sha256:2045a256902a6b7cd61330dd30a15767cc32bdf4f0fdea243ed7fe4a97c1f517",
+ "patchBytes": 489,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 2,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "0dd120528e6773e0dfc3a76bf4740870ba17fd32da59aae1b1abf7895ef676db",
+ "size": 586,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "41b5ca0d459b453de00b5d66ef8578e23bffad35792a069ee460eebd4282366e",
+ "size": 742,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573969869-70a183d32ae7d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 23.527167,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573969869-70a183d32ae7d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 18.968208,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [
+ {
+ "gradeVersion": "1.0",
+ "gradeId": "grade:48758757945eece3abe919926dea1c4a6c86124970b81753fd524d3ec8645061",
+ "graderId": "agent-alias-relation-grader",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "treeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-6FnmI7/workspace",
+ "alias-relation"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 30.828667,
+ "stdoutSha256": "sha256:4a3966d3a5a5052d2f32c8dc8a3aed53e4834383504caa9459c2442b66a0aded",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "alias-relation: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+ }
+ ],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 489,
+ "sha256": "sha256:2045a256902a6b7cd61330dd30a15767cc32bdf4f0fdea243ed7fe4a97c1f517"
+ },
+ {
+ "id": "grade-task-grader",
+ "path": "grade-task-grader.json",
+ "mediaType": "application/json",
+ "bytes": 1938,
+ "sha256": "sha256:af694ed2e5d45f345d7fc8b3360afb9733e4cc73b2589dde103e50cf11e7d9ef"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 3227,
+ "sha256": "sha256:e22e027e60a7c218cbf776c65ba16fc9a49abba8cc833e78b922d2378bd7e2ec"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 265,
+ "sha256": "sha256:a48c9c13252c885c6d147c5a877a8848442d2f66289648a6a48540ec686b8f9e"
+ }
+ ],
+ "verdict": "passed",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 0,
+ "semanticVerdict": "passed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r02/TASK.md b/research/agent-eval/results/claude/trials/alias-relation-r02/TASK.md
new file mode 100644
index 0000000..f4381c1
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r02/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, add alias Alicia that resolves to existing entity alice. Also add exactly one master relation with id alice-bob-friend, a alice, b bob, type friend, and label Friend. Do not create entities or modify existing names. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r02/agent.patch b/research/agent-eval/results/claude/trials/alias-relation-r02/agent.patch
new file mode 100644
index 0000000..15c5fad
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r02/agent.patch
@@ -0,0 +1,25 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -17,11 +17,20 @@
+ "aliases": {
+ "Alice": "alice",
++ "Alicia": "alice",
+ "Bob": "bob"
+ },
+ "relationTypes": {
+ "friend": {
+ "label": "Friend"
+ }
+ },
+- "relations": [],
++ "relations": [
++ {
++ "id": "alice-bob-friend",
++ "a": "alice",
++ "b": "bob",
++ "type": "friend",
++ "label": "Friend"
++ }
++ ],
+ "stages": [
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r02/grade-task-grader.json b/research/agent-eval/results/claude/trials/alias-relation-r02/grade-task-grader.json
new file mode 100644
index 0000000..6005dfe
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r02/grade-task-grader.json
@@ -0,0 +1,65 @@
+{
+ "gradeVersion": "1.0",
+ "gradeId": "grade:f5d1306ffae35d88e0886ec637631a974746321808bbc08870217785add2c655",
+ "graderId": "agent-alias-relation-grader",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "treeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "alias-relation"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 47.74475,
+ "stdoutSha256": "sha256:4a3966d3a5a5052d2f32c8dc8a3aed53e4834383504caa9459c2442b66a0aded",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "alias-relation: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r02/patch-audit.json b/research/agent-eval/results/claude/trials/alias-relation-r02/patch-audit.json
new file mode 100644
index 0000000..02863ec
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r02/patch-audit.json
@@ -0,0 +1,102 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:8a01a50d0fd78a7c20e595a0045d9cfbc3d84a4564860b56bce17033b1f2518b",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "applied",
+ "patchSha256": "sha256:f356bf37dc269359f3417f4e4c8ee02f7c4dbdf132dd14035f29cb70367a4bf3",
+ "patchBytes": 483,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "0dd120528e6773e0dfc3a76bf4740870ba17fd32da59aae1b1abf7895ef676db",
+ "size": 586,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "41b5ca0d459b453de00b5d66ef8578e23bffad35792a069ee460eebd4282366e",
+ "size": 742,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574014402-e529a66f77301.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 75.279291,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574014402-e529a66f77301.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 48.46525,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r02/task-run.json b/research/agent-eval/results/claude/trials/alias-relation-r02/task-run.json
new file mode 100644
index 0000000..435e0fd
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r02/task-run.json
@@ -0,0 +1,298 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:5752bc64b30328c65d022760d19df872a2d7d570418d6ae3b5edf0ed31b41608",
+ "task": {
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "taskFileSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630",
+ "sourceRevision": "alias-relation-fixture-v1",
+ "contractTreeSha256": "sha256:fffbe5c6cd3b100ea754dfc73b72bb75adae0ccd8ca789fb18c00ed68b2cfc53",
+ "runnerSnapshotBeforeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "runnerSnapshotFinalSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "alias-relation-r02"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r02/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r02/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 44366.553833,
+ "stdoutSha256": "sha256:a90f2d43f8ac3d4ac39ee02cd226a29bba024a1825e8f05467af3753c08da534",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1284,\"outputTokens\":6789,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.176209,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1284,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":6789,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.176209,\"durationApiMs\":43993,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1284,
+ "outputTokens": 6789,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.176209,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1284,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 6789,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.176209,
+ "durationApiMs": 43993,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:8a01a50d0fd78a7c20e595a0045d9cfbc3d84a4564860b56bce17033b1f2518b",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "applied",
+ "patchSha256": "sha256:f356bf37dc269359f3417f4e4c8ee02f7c4dbdf132dd14035f29cb70367a4bf3",
+ "patchBytes": 483,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "0dd120528e6773e0dfc3a76bf4740870ba17fd32da59aae1b1abf7895ef676db",
+ "size": 586,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "41b5ca0d459b453de00b5d66ef8578e23bffad35792a069ee460eebd4282366e",
+ "size": 742,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574014402-e529a66f77301.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 75.279291,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574014402-e529a66f77301.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 48.46525,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [
+ {
+ "gradeVersion": "1.0",
+ "gradeId": "grade:f5d1306ffae35d88e0886ec637631a974746321808bbc08870217785add2c655",
+ "graderId": "agent-alias-relation-grader",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "treeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-W2etOX/workspace",
+ "alias-relation"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 47.74475,
+ "stdoutSha256": "sha256:4a3966d3a5a5052d2f32c8dc8a3aed53e4834383504caa9459c2442b66a0aded",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "alias-relation: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+ }
+ ],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 483,
+ "sha256": "sha256:f356bf37dc269359f3417f4e4c8ee02f7c4dbdf132dd14035f29cb70367a4bf3"
+ },
+ {
+ "id": "grade-task-grader",
+ "path": "grade-task-grader.json",
+ "mediaType": "application/json",
+ "bytes": 1937,
+ "sha256": "sha256:eacbd965ce8936d17ae7bcab153c035da3f2ade0a2b8faded800be1b2d0e3d3b"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 3226,
+ "sha256": "sha256:8bf927a7163a8941e67e65b549c2aaaf978bcad9232812122250aa2d08e6f015"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 265,
+ "sha256": "sha256:a48c9c13252c885c6d147c5a877a8848442d2f66289648a6a48540ec686b8f9e"
+ }
+ ],
+ "verdict": "passed",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 0,
+ "semanticVerdict": "passed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r03/TASK.md b/research/agent-eval/results/claude/trials/alias-relation-r03/TASK.md
new file mode 100644
index 0000000..f4381c1
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r03/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, add alias Alicia that resolves to existing entity alice. Also add exactly one master relation with id alice-bob-friend, a alice, b bob, type friend, and label Friend. Do not create entities or modify existing names. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r03/agent.patch b/research/agent-eval/results/claude/trials/alias-relation-r03/agent.patch
new file mode 100644
index 0000000..ff7eccf
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r03/agent.patch
@@ -0,0 +1,26 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -11,11 +11,12 @@
+ },
+ "aliases": {
+ "Alice": "alice",
++ "Alicia": "alice",
+ "Bob": "bob"
+ },
+ "relationTypes": {
+ "friend": {
+ "label": "Friend"
+ }
+ },
+- "relations": [],
++ "relations": [
++ {
++ "id": "alice-bob-friend",
++ "a": "alice",
++ "b": "bob",
++ "type": "friend",
++ "label": "Friend"
++ }
++ ],
+ "stages": [
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r03/patch-audit.json b/research/agent-eval/results/claude/trials/alias-relation-r03/patch-audit.json
new file mode 100644
index 0000000..61366d0
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r03/patch-audit.json
@@ -0,0 +1,87 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:3d130ebe764c4a36cf67102e04a369f1e56cce7141353b97109c3706b04d87d7",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "rejected",
+ "patchSha256": "sha256:5d251043d5b64a4c55b9677c59c4d99f0d343140a638738fe3f27741f1195e9c",
+ "patchBytes": 489,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574024447-85bba020679ec.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-omgCH1/workspace",
+ "exitCode": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 33.42575,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:8e43234eaee96c9af409afb02af9666486ade6b792af89b79edf8a553711bc25",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574024447-85bba020679ec.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-omgCH1/workspace",
+ "status": 1,
+ "exitCode": 1,
+ "exit": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 33.42575,
+ "duration": 33.42575,
+ "stdout": "",
+ "stderr": "error: patch failed: lib/data/data.json:11\nerror: lib/data/data.json: patch does not apply\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r03/task-run.json b/research/agent-eval/results/claude/trials/alias-relation-r03/task-run.json
new file mode 100644
index 0000000..6c2b95e
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r03/task-run.json
@@ -0,0 +1,211 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:f198854b1a316d4f255f970c87ed2763e7938159828a18d060c647a0abcfecad",
+ "task": {
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "taskFileSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630",
+ "sourceRevision": "alias-relation-fixture-v1",
+ "contractTreeSha256": "sha256:fffbe5c6cd3b100ea754dfc73b72bb75adae0ccd8ca789fb18c00ed68b2cfc53",
+ "runnerSnapshotBeforeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "runnerSnapshotFinalSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "alias-relation-r03"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-omgCH1/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r03/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r03/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-omgCH1/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 9756.787833,
+ "stdoutSha256": "sha256:098f72c2bbfbb00bd46948c47302c3b8ea6823c748f35998091cfaf6c609d9a2",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1285,\"outputTokens\":454,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.017839,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1285,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":454,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.017839,\"durationApiMs\":8821,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1285,
+ "outputTokens": 454,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.017839,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1285,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 454,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.017839,
+ "durationApiMs": 8821,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:3d130ebe764c4a36cf67102e04a369f1e56cce7141353b97109c3706b04d87d7",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "rejected",
+ "patchSha256": "sha256:5d251043d5b64a4c55b9677c59c4d99f0d343140a638738fe3f27741f1195e9c",
+ "patchBytes": 489,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574024447-85bba020679ec.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-omgCH1/workspace",
+ "exitCode": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 33.42575,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:8e43234eaee96c9af409afb02af9666486ade6b792af89b79edf8a553711bc25",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574024447-85bba020679ec.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-omgCH1/workspace",
+ "status": 1,
+ "exitCode": 1,
+ "exit": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 33.42575,
+ "duration": 33.42575,
+ "stdout": "",
+ "stderr": "error: patch failed: lib/data/data.json:11\nerror: lib/data/data.json: patch does not apply\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 489,
+ "sha256": "sha256:5d251043d5b64a4c55b9677c59c4d99f0d343140a638738fe3f27741f1195e9c"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 2709,
+ "sha256": "sha256:e70b0f179c783daca4003ccd3c44b89aa7818108c6e0423d21a35d99f5db3ab6"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 265,
+ "sha256": "sha256:a48c9c13252c885c6d147c5a877a8848442d2f66289648a6a48540ec686b8f9e"
+ }
+ ],
+ "verdict": "patch-invalid",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 1,
+ "semanticVerdict": "patch-invalid",
+ "message": "git apply --check rejected the patch"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r04/TASK.md b/research/agent-eval/results/claude/trials/alias-relation-r04/TASK.md
new file mode 100644
index 0000000..f4381c1
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r04/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, add alias Alicia that resolves to existing entity alice. Also add exactly one master relation with id alice-bob-friend, a alice, b bob, type friend, and label Friend. Do not create entities or modify existing names. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r04/agent.patch b/research/agent-eval/results/claude/trials/alias-relation-r04/agent.patch
new file mode 100644
index 0000000..4d09982
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r04/agent.patch
@@ -0,0 +1,31 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -15,9 +15,10 @@
+ }
+ },
+ "aliases": {
+ "Alice": "alice",
++ "Alicia": "alice",
+ "Bob": "bob"
+ },
+ "relationTypes": {
+ "friend": {
+ "label": "Friend"
+@@ -23,7 +24,15 @@
+ "label": "Friend"
+ }
+ },
+- "relations": [],
++ "relations": [
++ {
++ "id": "alice-bob-friend",
++ "a": "alice",
++ "b": "bob",
++ "type": "friend",
++ "label": "Friend"
++ }
++ ],
+ "stages": [
+ {
+ "key": "main",
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r04/patch-audit.json b/research/agent-eval/results/claude/trials/alias-relation-r04/patch-audit.json
new file mode 100644
index 0000000..9bdc173
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r04/patch-audit.json
@@ -0,0 +1,87 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:58962824b347846848ef207ee264a07b7235d9a41d4bc8c0886b9b8df02978a2",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "rejected",
+ "patchSha256": "sha256:fa5264e9c6a06ab351e3cbf709e1cd592bba6137f5bee92b6e41f71715deff1a",
+ "patchBytes": 568,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 2,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574098232-2bf4dc96cdd56.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-2eg6Tq/workspace",
+ "exitCode": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 20.418917,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:48aa97bfde5200caeb85320f2011a37ecad6246ce3b5d6db6f757d676e115ad2",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574098232-2bf4dc96cdd56.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-2eg6Tq/workspace",
+ "status": 1,
+ "exitCode": 1,
+ "exit": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 20.418917,
+ "duration": 20.418917,
+ "stdout": "",
+ "stderr": "error: patch failed: lib/data/data.json:23\nerror: lib/data/data.json: patch does not apply\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r04/task-run.json b/research/agent-eval/results/claude/trials/alias-relation-r04/task-run.json
new file mode 100644
index 0000000..15da105
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r04/task-run.json
@@ -0,0 +1,211 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:6c4a7a66037f3e6a9cf0029744d4fc930e1cb95a7da420acd8e1a6552b622d09",
+ "task": {
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "taskFileSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630",
+ "sourceRevision": "alias-relation-fixture-v1",
+ "contractTreeSha256": "sha256:fffbe5c6cd3b100ea754dfc73b72bb75adae0ccd8ca789fb18c00ed68b2cfc53",
+ "runnerSnapshotBeforeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "runnerSnapshotFinalSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "alias-relation-r04"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-2eg6Tq/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r04/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r04/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-2eg6Tq/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 73701.680542,
+ "stdoutSha256": "sha256:199c2c187c96b09b5e70697152d66d1335a92ceea3f7a9b4d677bf9e26c63339",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1286,\"outputTokens\":5806,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.151644,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1286,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":5806,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.151644,\"durationApiMs\":72771,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1286,
+ "outputTokens": 5806,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.151644,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1286,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 5806,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.151644,
+ "durationApiMs": 72771,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:58962824b347846848ef207ee264a07b7235d9a41d4bc8c0886b9b8df02978a2",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "rejected",
+ "patchSha256": "sha256:fa5264e9c6a06ab351e3cbf709e1cd592bba6137f5bee92b6e41f71715deff1a",
+ "patchBytes": 568,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 2,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574098232-2bf4dc96cdd56.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-2eg6Tq/workspace",
+ "exitCode": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 20.418917,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:48aa97bfde5200caeb85320f2011a37ecad6246ce3b5d6db6f757d676e115ad2",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574098232-2bf4dc96cdd56.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-2eg6Tq/workspace",
+ "status": 1,
+ "exitCode": 1,
+ "exit": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 20.418917,
+ "duration": 20.418917,
+ "stdout": "",
+ "stderr": "error: patch failed: lib/data/data.json:23\nerror: lib/data/data.json: patch does not apply\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 568,
+ "sha256": "sha256:fa5264e9c6a06ab351e3cbf709e1cd592bba6137f5bee92b6e41f71715deff1a"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 2712,
+ "sha256": "sha256:1b31172ae68116732dc082343edfc9421d3af9e811492161c637f4e36b96080f"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 265,
+ "sha256": "sha256:a48c9c13252c885c6d147c5a877a8848442d2f66289648a6a48540ec686b8f9e"
+ }
+ ],
+ "verdict": "patch-invalid",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 1,
+ "semanticVerdict": "patch-invalid",
+ "message": "git apply --check rejected the patch"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r05/TASK.md b/research/agent-eval/results/claude/trials/alias-relation-r05/TASK.md
new file mode 100644
index 0000000..f4381c1
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r05/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, add alias Alicia that resolves to existing entity alice. Also add exactly one master relation with id alice-bob-friend, a alice, b bob, type friend, and label Friend. Do not create entities or modify existing names. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r05/agent.patch b/research/agent-eval/results/claude/trials/alias-relation-r05/agent.patch
new file mode 100644
index 0000000..02de8fd
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r05/agent.patch
@@ -0,0 +1,29 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -15,7 +15,8 @@
+ }
+ },
+ "aliases": {
+ "Alice": "alice",
++ "Alicia": "alice",
+ "Bob": "bob"
+ },
+ "relationTypes": {
+@@ -23,7 +24,15 @@
+ "label": "Friend"
+ }
+ },
+- "relations": [],
++ "relations": [
++ {
++ "id": "alice-bob-friend",
++ "a": "alice",
++ "b": "bob",
++ "type": "friend",
++ "label": "Friend"
++ }
++ ],
+ "stages": [
+ {
+ "key": "main",
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r05/grade-task-grader.json b/research/agent-eval/results/claude/trials/alias-relation-r05/grade-task-grader.json
new file mode 100644
index 0000000..bb0c340
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r05/grade-task-grader.json
@@ -0,0 +1,65 @@
+{
+ "gradeVersion": "1.0",
+ "gradeId": "grade:123a7cb648054d988999f93754d90d13fc90faecf90da4a7e2c07cf168a4f70c",
+ "graderId": "agent-alias-relation-grader",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "treeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "alias-relation"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 30.407375,
+ "stdoutSha256": "sha256:4a3966d3a5a5052d2f32c8dc8a3aed53e4834383504caa9459c2442b66a0aded",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "alias-relation: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r05/patch-audit.json b/research/agent-eval/results/claude/trials/alias-relation-r05/patch-audit.json
new file mode 100644
index 0000000..f2a41e1
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r05/patch-audit.json
@@ -0,0 +1,102 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:83ef939bf7d18de346035e9515ca83d6422ac6d4e41d920a97878422a7249610",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "applied",
+ "patchSha256": "sha256:747f47bb2d06693339aa25fd1bbecafe07858afa3997c3f882e0a4eb58d6ff88",
+ "patchBytes": 525,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 2,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "0dd120528e6773e0dfc3a76bf4740870ba17fd32da59aae1b1abf7895ef676db",
+ "size": 586,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "41b5ca0d459b453de00b5d66ef8578e23bffad35792a069ee460eebd4282366e",
+ "size": 742,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574204129-3ae3332958316.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 26.777375,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574204129-3ae3332958316.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.642125,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/alias-relation-r05/task-run.json b/research/agent-eval/results/claude/trials/alias-relation-r05/task-run.json
new file mode 100644
index 0000000..231a7d5
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/alias-relation-r05/task-run.json
@@ -0,0 +1,298 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:9c06d2f4dcd88ef9c0894e318e07d4e61331b53315b665487b9bb4a441fa921b",
+ "task": {
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "taskFileSha256": "sha256:ac33ff5e0049324b56141ea0687442ed5e344fb6305390f8257b02f2cd17a630",
+ "sourceRevision": "alias-relation-fixture-v1",
+ "contractTreeSha256": "sha256:fffbe5c6cd3b100ea754dfc73b72bb75adae0ccd8ca789fb18c00ed68b2cfc53",
+ "runnerSnapshotBeforeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "runnerSnapshotFinalSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "alias-relation-r05"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r05/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/alias-relation-r05/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 105836.185208,
+ "stdoutSha256": "sha256:45844b49a4ea873a8b7713e4e17c35a212e4920a461ede6132deede722b35f52",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1285,\"outputTokens\":9192,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.236289,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1285,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":9192,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.236289,\"durationApiMs\":105475,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1285,
+ "outputTokens": 9192,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.236289,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1285,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9192,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.236289,
+ "durationApiMs": 105475,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:83ef939bf7d18de346035e9515ca83d6422ac6d4e41d920a97878422a7249610",
+ "kind": "patch-audit",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "status": "applied",
+ "patchSha256": "sha256:747f47bb2d06693339aa25fd1bbecafe07858afa3997c3f882e0a4eb58d6ff88",
+ "patchBytes": 525,
+ "beforeTreeSha256": "sha256:6243526415fa11e029c8205e3ee3ddb2f0cb824e7e561c733480c3c8ecabb6c5",
+ "afterTreeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 2,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "0dd120528e6773e0dfc3a76bf4740870ba17fd32da59aae1b1abf7895ef676db",
+ "size": 586,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "41b5ca0d459b453de00b5d66ef8578e23bffad35792a069ee460eebd4282366e",
+ "size": 742,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574204129-3ae3332958316.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 26.777375,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574204129-3ae3332958316.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.642125,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [
+ {
+ "gradeVersion": "1.0",
+ "gradeId": "grade:123a7cb648054d988999f93754d90d13fc90faecf90da4a7e2c07cf168a4f70c",
+ "graderId": "agent-alias-relation-grader",
+ "taskId": "sha256:295709a674170fcf6188e9ae793fbd2f42e8f57c95287f3fc27a0e5556f1e118",
+ "treeSha256": "sha256:cfb2b07547916e565b567b263a219e91d34738cdfce5ceeb5957ca974260102d",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-EkLU1A/workspace",
+ "alias-relation"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 30.407375,
+ "stdoutSha256": "sha256:4a3966d3a5a5052d2f32c8dc8a3aed53e4834383504caa9459c2442b66a0aded",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "alias-relation: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+ }
+ ],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 525,
+ "sha256": "sha256:747f47bb2d06693339aa25fd1bbecafe07858afa3997c3f882e0a4eb58d6ff88"
+ },
+ {
+ "id": "grade-task-grader",
+ "path": "grade-task-grader.json",
+ "mediaType": "application/json",
+ "bytes": 1938,
+ "sha256": "sha256:a8f6a64d8ffdadeca391ea33d0714af070641418a62811dc7880430434670515"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 3227,
+ "sha256": "sha256:15b7c1092b97afa5ce5502c82f97c7677100fa76f9f96979e3d1de751344bbdd"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 265,
+ "sha256": "sha256:a48c9c13252c885c6d147c5a877a8848442d2f66289648a6a48540ec686b8f9e"
+ }
+ ],
+ "verdict": "passed",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 0,
+ "semanticVerdict": "passed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r01/TASK.md b/research/agent-eval/results/claude/trials/field-update-r01/TASK.md
new file mode 100644
index 0000000..163fbf2
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r01/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, change only entities.alice.summary from summary-old to exactly summary-new. Preserve the valid Data Pack shape and every unrelated value. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/field-update-r01/agent.patch b/research/agent-eval/results/claude/trials/field-update-r01/agent.patch
new file mode 100644
index 0000000..9aad4ae
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r01/agent.patch
@@ -0,0 +1,8 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -8,7 +8,7 @@
+ "kind": "person",
+ "name": "Alice",
+ "summary": "summary-old"
+- "summary": "summary-new"
diff --git a/research/agent-eval/results/claude/trials/field-update-r01/patch-audit.json b/research/agent-eval/results/claude/trials/field-update-r01/patch-audit.json
new file mode 100644
index 0000000..0302c75
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r01/patch-audit.json
@@ -0,0 +1,87 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:1a9a7582d7b0bb67b887c5473ca55af0908657d6bb17eb8665dfb13cf5542153",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "rejected",
+ "patchSha256": "sha256:4206f16fb681c139719b49e6d94abbadf93959839702dc360bce26aa058ff61f",
+ "patchBytes": 232,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573843472-ce2fa6682c38e.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-jm9X14/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.504417,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:97e03fbd499c7f04af57d26fa8bda99067c2d301e271adc5f1462c40cd89bba3",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573843472-ce2fa6682c38e.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-jm9X14/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.504417,
+ "duration": 14.504417,
+ "stdout": "",
+ "stderr": "error: corrupt patch at line 9\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r01/task-run.json b/research/agent-eval/results/claude/trials/field-update-r01/task-run.json
new file mode 100644
index 0000000..b587cd0
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r01/task-run.json
@@ -0,0 +1,211 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:eda458f8bd0740b373639a702e700e17e55e8fb73e2dda8c5b0b187a3de0fe7c",
+ "task": {
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "taskFileSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a",
+ "sourceRevision": "field-update-fixture-v1",
+ "contractTreeSha256": "sha256:5f3c536db01e9f4c2c1dbaabf9216bb532afc6f067ea18a7d13d89255105b35a",
+ "runnerSnapshotBeforeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "runnerSnapshotFinalSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "field-update-r01"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-jm9X14/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r01/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r01/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-jm9X14/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 3840.298,
+ "stdoutSha256": "sha256:a364c42cc6397efb63db1521e33ada563734cc4f4c33600fa8a7689090d4076e",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1205,\"outputTokens\":135,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.009463999999999998,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1205,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":135,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.009463999999999998,\"durationApiMs\":2651,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 135,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.009463999999999998,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 135,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.009463999999999998,
+ "durationApiMs": 2651,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:1a9a7582d7b0bb67b887c5473ca55af0908657d6bb17eb8665dfb13cf5542153",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "rejected",
+ "patchSha256": "sha256:4206f16fb681c139719b49e6d94abbadf93959839702dc360bce26aa058ff61f",
+ "patchBytes": 232,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573843472-ce2fa6682c38e.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-jm9X14/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.504417,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:97e03fbd499c7f04af57d26fa8bda99067c2d301e271adc5f1462c40cd89bba3",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573843472-ce2fa6682c38e.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-jm9X14/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.504417,
+ "duration": 14.504417,
+ "stdout": "",
+ "stderr": "error: corrupt patch at line 9\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 232,
+ "sha256": "sha256:4206f16fb681c139719b49e6d94abbadf93959839702dc360bce26aa058ff61f"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 2659,
+ "sha256": "sha256:6db3ee7991dbedd397d84346be1518c7b24b2c600f6d1bee00a953f0025d84cc"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 187,
+ "sha256": "sha256:21f3f361c74972d195f556886659d9f83a5f002e17ea3ad0dd8bc004a46d7e62"
+ }
+ ],
+ "verdict": "patch-invalid",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 1,
+ "semanticVerdict": "patch-invalid",
+ "message": "git apply --check rejected the patch"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r02/TASK.md b/research/agent-eval/results/claude/trials/field-update-r02/TASK.md
new file mode 100644
index 0000000..163fbf2
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r02/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, change only entities.alice.summary from summary-old to exactly summary-new. Preserve the valid Data Pack shape and every unrelated value. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/field-update-r02/agent.patch b/research/agent-eval/results/claude/trials/field-update-r02/agent.patch
new file mode 100644
index 0000000..f477cef
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r02/agent.patch
@@ -0,0 +1,11 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -11,7 +11,7 @@
+ "kind": "person",
+ "name": "Alice",
+- "summary": "summary-old"
++ "summary": "summary-new"
+ }
+ },
+ "aliases": {
diff --git a/research/agent-eval/results/claude/trials/field-update-r02/patch-audit.json b/research/agent-eval/results/claude/trials/field-update-r02/patch-audit.json
new file mode 100644
index 0000000..a54a60b
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r02/patch-audit.json
@@ -0,0 +1,87 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:cf9968dd9d7faa59142cf473b9195d1e61e72b1ea00b42cf2d3767993c8ce3d6",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "rejected",
+ "patchSha256": "sha256:fee82b559e7b8e1ab73aed52cd41db243dd6d345d8bff06296242260baf2c882",
+ "patchBytes": 263,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573846312-e650bd822cd15.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-j3ZSMo/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.829958,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:a605710393416cbb2aeaaf05e61e89ae5bd80941edf6faff1444c7d98e996285",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573846312-e650bd822cd15.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-j3ZSMo/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.829958,
+ "duration": 14.829958,
+ "stdout": "",
+ "stderr": "error: corrupt patch at line 12\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r02/task-run.json b/research/agent-eval/results/claude/trials/field-update-r02/task-run.json
new file mode 100644
index 0000000..6300bac
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r02/task-run.json
@@ -0,0 +1,211 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:309918a0fb60925f85b679ec333015c77b33eec76d0030b7dcdb9624b5ea525b",
+ "task": {
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "taskFileSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a",
+ "sourceRevision": "field-update-fixture-v1",
+ "contractTreeSha256": "sha256:5f3c536db01e9f4c2c1dbaabf9216bb532afc6f067ea18a7d13d89255105b35a",
+ "runnerSnapshotBeforeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "runnerSnapshotFinalSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "field-update-r02"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-j3ZSMo/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r02/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r02/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-j3ZSMo/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 2788.293459,
+ "stdoutSha256": "sha256:debaf11137c9c8e5b17bfec14924d1f095c95e5979552694a6386c1481f9a1ac",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1205,\"outputTokens\":167,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.010263999999999999,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1205,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":167,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.010263999999999999,\"durationApiMs\":2453,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 167,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.010263999999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 167,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.010263999999999999,
+ "durationApiMs": 2453,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:cf9968dd9d7faa59142cf473b9195d1e61e72b1ea00b42cf2d3767993c8ce3d6",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "rejected",
+ "patchSha256": "sha256:fee82b559e7b8e1ab73aed52cd41db243dd6d345d8bff06296242260baf2c882",
+ "patchBytes": 263,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573846312-e650bd822cd15.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-j3ZSMo/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.829958,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:a605710393416cbb2aeaaf05e61e89ae5bd80941edf6faff1444c7d98e996285",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573846312-e650bd822cd15.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-j3ZSMo/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.829958,
+ "duration": 14.829958,
+ "stdout": "",
+ "stderr": "error: corrupt patch at line 12\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 263,
+ "sha256": "sha256:fee82b559e7b8e1ab73aed52cd41db243dd6d345d8bff06296242260baf2c882"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 2660,
+ "sha256": "sha256:c99a0a726c360e331267def9b9695dbbfaa6dd17c82617cfaa553537e426f63a"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 187,
+ "sha256": "sha256:21f3f361c74972d195f556886659d9f83a5f002e17ea3ad0dd8bc004a46d7e62"
+ }
+ ],
+ "verdict": "patch-invalid",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 1,
+ "semanticVerdict": "patch-invalid",
+ "message": "git apply --check rejected the patch"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r03/TASK.md b/research/agent-eval/results/claude/trials/field-update-r03/TASK.md
new file mode 100644
index 0000000..163fbf2
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r03/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, change only entities.alice.summary from summary-old to exactly summary-new. Preserve the valid Data Pack shape and every unrelated value. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/field-update-r03/agent.patch b/research/agent-eval/results/claude/trials/field-update-r03/agent.patch
new file mode 100644
index 0000000..cd53c71
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r03/agent.patch
@@ -0,0 +1,13 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -8,7 +8,7 @@
+ "entities": {
+ "alice": {
+ "kind": "person",
+ "name": "Alice",
+- "summary": "summary-old"
++ "summary": "summary-new"
+ }
+ },
+ "aliases": {
diff --git a/research/agent-eval/results/claude/trials/field-update-r03/grade-task-grader.json b/research/agent-eval/results/claude/trials/field-update-r03/grade-task-grader.json
new file mode 100644
index 0000000..1fab7d5
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r03/grade-task-grader.json
@@ -0,0 +1,65 @@
+{
+ "gradeVersion": "1.0",
+ "gradeId": "grade:f98d2e3cb95ac4032d756cf88cb8054e1c54666d11b6df5e3a2b0bc38e663da0",
+ "graderId": "agent-field-update-grader",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "treeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "field-update"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 27.675458,
+ "stdoutSha256": "sha256:fb2d6d6ccc2c7839e527f75052fa856b3e0b5c9aff34f424bd69337e5380f1c0",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "field-update: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r03/patch-audit.json b/research/agent-eval/results/claude/trials/field-update-r03/patch-audit.json
new file mode 100644
index 0000000..891165e
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r03/patch-audit.json
@@ -0,0 +1,102 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:09832a40ffce2003ad79ca8e833c13ee10356010cc518b65662997d1148ee2af",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "applied",
+ "patchSha256": "sha256:67185c0bfc28b4d9f52efb2c3187b57cf4829ab84b2c854dd36a46531aadfd03",
+ "patchBytes": 294,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "4af3f1f6eb137aed5e32146530a068601252760103e63e400f1ba89aadb5bd5e",
+ "size": 399,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "7df813f5c953407911e67c0d882560331cdde9c29bf4cc968340f278ebbc72e8",
+ "size": 399,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573866872-eb5ad7864bb0d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.961333,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573866872-eb5ad7864bb0d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.602291,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r03/task-run.json b/research/agent-eval/results/claude/trials/field-update-r03/task-run.json
new file mode 100644
index 0000000..c96a1c8
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r03/task-run.json
@@ -0,0 +1,298 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:6ebe4b699f124bdb070f4ec2b44f8a2533dcc0b2951ced11677219ab27cb3233",
+ "task": {
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "taskFileSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a",
+ "sourceRevision": "field-update-fixture-v1",
+ "contractTreeSha256": "sha256:5f3c536db01e9f4c2c1dbaabf9216bb532afc6f067ea18a7d13d89255105b35a",
+ "runnerSnapshotBeforeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "runnerSnapshotFinalSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "field-update-r03"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r03/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r03/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 20519.502583,
+ "stdoutSha256": "sha256:6a599900952686cd82ee9a71f32079bbdceddb8b98255b1f5f39032edb5879cd",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1204,\"outputTokens\":1163,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.035159,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1204,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":1163,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.035159,\"durationApiMs\":20145,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1204,
+ "outputTokens": 1163,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.035159,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1204,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 1163,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.035159,
+ "durationApiMs": 20145,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:09832a40ffce2003ad79ca8e833c13ee10356010cc518b65662997d1148ee2af",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "applied",
+ "patchSha256": "sha256:67185c0bfc28b4d9f52efb2c3187b57cf4829ab84b2c854dd36a46531aadfd03",
+ "patchBytes": 294,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "4af3f1f6eb137aed5e32146530a068601252760103e63e400f1ba89aadb5bd5e",
+ "size": 399,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "7df813f5c953407911e67c0d882560331cdde9c29bf4cc968340f278ebbc72e8",
+ "size": 399,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573866872-eb5ad7864bb0d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.961333,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573866872-eb5ad7864bb0d.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 14.602291,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [
+ {
+ "gradeVersion": "1.0",
+ "gradeId": "grade:f98d2e3cb95ac4032d756cf88cb8054e1c54666d11b6df5e3a2b0bc38e663da0",
+ "graderId": "agent-field-update-grader",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "treeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-i9bvKe/workspace",
+ "field-update"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 27.675458,
+ "stdoutSha256": "sha256:fb2d6d6ccc2c7839e527f75052fa856b3e0b5c9aff34f424bd69337e5380f1c0",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "field-update: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+ }
+ ],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 294,
+ "sha256": "sha256:67185c0bfc28b4d9f52efb2c3187b57cf4829ab84b2c854dd36a46531aadfd03"
+ },
+ {
+ "id": "grade-task-grader",
+ "path": "grade-task-grader.json",
+ "mediaType": "application/json",
+ "bytes": 1932,
+ "sha256": "sha256:346f5bc3c35e3a1577b15d1847a600e8b73bd4f21ba61e96a3a37e62c8257a51"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 3227,
+ "sha256": "sha256:440dad5f8e9bcd9fa24bdad68d4fd5c5d40f1e27aa81199ff3e0ec7721b98cdf"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 187,
+ "sha256": "sha256:21f3f361c74972d195f556886659d9f83a5f002e17ea3ad0dd8bc004a46d7e62"
+ }
+ ],
+ "verdict": "passed",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 0,
+ "semanticVerdict": "passed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r04/TASK.md b/research/agent-eval/results/claude/trials/field-update-r04/TASK.md
new file mode 100644
index 0000000..163fbf2
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r04/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, change only entities.alice.summary from summary-old to exactly summary-new. Preserve the valid Data Pack shape and every unrelated value. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/field-update-r04/agent.patch b/research/agent-eval/results/claude/trials/field-update-r04/agent.patch
new file mode 100644
index 0000000..73247f7
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r04/agent.patch
@@ -0,0 +1,12 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -10,7 +10,7 @@
+ "alice": {
+ "kind": "person",
+ "name": "Alice",
+- "summary": "summary-old"
++ "summary": "summary-new"
+ }
+ },
+ "aliases": {
diff --git a/research/agent-eval/results/claude/trials/field-update-r04/grade-task-grader.json b/research/agent-eval/results/claude/trials/field-update-r04/grade-task-grader.json
new file mode 100644
index 0000000..85ebfb9
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r04/grade-task-grader.json
@@ -0,0 +1,65 @@
+{
+ "gradeVersion": "1.0",
+ "gradeId": "grade:0e788c1c0db07b25ab5e5145a2e69a76cefffa2eb587d872690645909258b450",
+ "graderId": "agent-field-update-grader",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "treeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "field-update"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 29.967209,
+ "stdoutSha256": "sha256:fb2d6d6ccc2c7839e527f75052fa856b3e0b5c9aff34f424bd69337e5380f1c0",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "field-update: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r04/patch-audit.json b/research/agent-eval/results/claude/trials/field-update-r04/patch-audit.json
new file mode 100644
index 0000000..1c3cad4
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r04/patch-audit.json
@@ -0,0 +1,102 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:56145345d02a45ebeb1e7cb0acb509c86264b91d34cace47fc3512305dd3af77",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "applied",
+ "patchSha256": "sha256:cbd9ba5c768e56ebf6ed8780c11d7563daa7cffff8c7ba241dca1f5766f05410",
+ "patchBytes": 279,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "4af3f1f6eb137aed5e32146530a068601252760103e63e400f1ba89aadb5bd5e",
+ "size": 399,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "7df813f5c953407911e67c0d882560331cdde9c29bf4cc968340f278ebbc72e8",
+ "size": 399,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573869620-3b07686c1a09a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 21.3615,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573869620-3b07686c1a09a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 18.145375,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r04/task-run.json b/research/agent-eval/results/claude/trials/field-update-r04/task-run.json
new file mode 100644
index 0000000..28293c3
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r04/task-run.json
@@ -0,0 +1,298 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:73c5544c5dc6f6c78c4320e2888567420ae338d827a2a39e9b86f17b1c4c2f33",
+ "task": {
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "taskFileSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a",
+ "sourceRevision": "field-update-fixture-v1",
+ "contractTreeSha256": "sha256:5f3c536db01e9f4c2c1dbaabf9216bb532afc6f067ea18a7d13d89255105b35a",
+ "runnerSnapshotBeforeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "runnerSnapshotFinalSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "field-update-r04"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r04/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r04/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 2629.936958,
+ "stdoutSha256": "sha256:9616d959fc25a261214dfa8291672bce8a0dcdd62f24dc6ac442018c07832fd9",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1204,\"outputTokens\":162,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.010133999999999999,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1204,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":162,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.010133999999999999,\"durationApiMs\":2291,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1204,
+ "outputTokens": 162,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.010133999999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1204,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 162,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.010133999999999999,
+ "durationApiMs": 2291,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:56145345d02a45ebeb1e7cb0acb509c86264b91d34cace47fc3512305dd3af77",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "applied",
+ "patchSha256": "sha256:cbd9ba5c768e56ebf6ed8780c11d7563daa7cffff8c7ba241dca1f5766f05410",
+ "patchBytes": 279,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "4af3f1f6eb137aed5e32146530a068601252760103e63e400f1ba89aadb5bd5e",
+ "size": 399,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "7df813f5c953407911e67c0d882560331cdde9c29bf4cc968340f278ebbc72e8",
+ "size": 399,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573869620-3b07686c1a09a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 21.3615,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573869620-3b07686c1a09a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 18.145375,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [
+ {
+ "gradeVersion": "1.0",
+ "gradeId": "grade:0e788c1c0db07b25ab5e5145a2e69a76cefffa2eb587d872690645909258b450",
+ "graderId": "agent-field-update-grader",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "treeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-yLObSy/workspace",
+ "field-update"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 29.967209,
+ "stdoutSha256": "sha256:fb2d6d6ccc2c7839e527f75052fa856b3e0b5c9aff34f424bd69337e5380f1c0",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "field-update: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+ }
+ ],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 279,
+ "sha256": "sha256:cbd9ba5c768e56ebf6ed8780c11d7563daa7cffff8c7ba241dca1f5766f05410"
+ },
+ {
+ "id": "grade-task-grader",
+ "path": "grade-task-grader.json",
+ "mediaType": "application/json",
+ "bytes": 1932,
+ "sha256": "sha256:f07b2949edd8df3077b00fd9d95a4925f1796f73df17042e400b0c43605cb21f"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 3225,
+ "sha256": "sha256:e688c2da94eee5595ad0b01e0e474d2de997f8d326bfe917ba642540a76bc851"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 187,
+ "sha256": "sha256:21f3f361c74972d195f556886659d9f83a5f002e17ea3ad0dd8bc004a46d7e62"
+ }
+ ],
+ "verdict": "passed",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 0,
+ "semanticVerdict": "passed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r05/TASK.md b/research/agent-eval/results/claude/trials/field-update-r05/TASK.md
new file mode 100644
index 0000000..163fbf2
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r05/TASK.md
@@ -0,0 +1 @@
+In lib/data/data.json, change only entities.alice.summary from summary-old to exactly summary-new. Preserve the valid Data Pack shape and every unrelated value. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/field-update-r05/agent.patch b/research/agent-eval/results/claude/trials/field-update-r05/agent.patch
new file mode 100644
index 0000000..b7ba36e
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r05/agent.patch
@@ -0,0 +1,12 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -8,7 +8,7 @@
+ "alice": {
+ "kind": "person",
+ "name": "Alice",
+- "summary": "summary-old"
++ "summary": "summary-new"
+ }
+ },
+ "aliases": {
diff --git a/research/agent-eval/results/claude/trials/field-update-r05/grade-task-grader.json b/research/agent-eval/results/claude/trials/field-update-r05/grade-task-grader.json
new file mode 100644
index 0000000..78834e1
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r05/grade-task-grader.json
@@ -0,0 +1,65 @@
+{
+ "gradeVersion": "1.0",
+ "gradeId": "grade:a6f4b3fc7af28f082c443a5a75990ef242790b295e5de8fd0c38de92149221f1",
+ "graderId": "agent-field-update-grader",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "treeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "field-update"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 33.318292,
+ "stdoutSha256": "sha256:fb2d6d6ccc2c7839e527f75052fa856b3e0b5c9aff34f424bd69337e5380f1c0",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "field-update: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r05/patch-audit.json b/research/agent-eval/results/claude/trials/field-update-r05/patch-audit.json
new file mode 100644
index 0000000..d26186f
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r05/patch-audit.json
@@ -0,0 +1,102 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:c8ffb51ba506b374e828e6fc2e065e4c4ccb233b304d8f7059d2d8c79d601ed0",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "applied",
+ "patchSha256": "sha256:8c5deb62c08c076632076d2296e1390fb414f0e18edf78255f777d1903671adc",
+ "patchBytes": 277,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "4af3f1f6eb137aed5e32146530a068601252760103e63e400f1ba89aadb5bd5e",
+ "size": 399,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "7df813f5c953407911e67c0d882560331cdde9c29bf4cc968340f278ebbc72e8",
+ "size": 399,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573904842-a233d4e4dc065.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 20.609917,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573904842-a233d4e4dc065.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 15.667833,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/field-update-r05/task-run.json b/research/agent-eval/results/claude/trials/field-update-r05/task-run.json
new file mode 100644
index 0000000..426ea10
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/field-update-r05/task-run.json
@@ -0,0 +1,298 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:528e87fb2aa72f2ccadb88f732444149a30535c80fc1eff8692c2f6c251a9873",
+ "task": {
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "taskFileSha256": "sha256:27b37bf28fd1fcd4e0d160caef426afbaf27f9a4258e3e92b7795609b249798a",
+ "sourceRevision": "field-update-fixture-v1",
+ "contractTreeSha256": "sha256:5f3c536db01e9f4c2c1dbaabf9216bb532afc6f067ea18a7d13d89255105b35a",
+ "runnerSnapshotBeforeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "runnerSnapshotFinalSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "field-update-r05"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r05/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/field-update-r05/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 35078.472834,
+ "stdoutSha256": "sha256:8a65c268225575cbfd1dfafdc5309a3b954e4b9406168da8da54e857e46cf201",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":1205,\"outputTokens\":2639,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.07206399999999999,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":1205,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":2639,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.07206399999999999,\"durationApiMs\":34735,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 1205,
+ "outputTokens": 2639,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.07206399999999999,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 1205,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 2639,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.07206399999999999,
+ "durationApiMs": 34735,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:c8ffb51ba506b374e828e6fc2e065e4c4ccb233b304d8f7059d2d8c79d601ed0",
+ "kind": "patch-audit",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "status": "applied",
+ "patchSha256": "sha256:8c5deb62c08c076632076d2296e1390fb414f0e18edf78255f777d1903671adc",
+ "patchBytes": 277,
+ "beforeTreeSha256": "sha256:793d34d30db2b23ad60b518e1ef2ba63af8a1731be03baa813eb0a5730bbf76e",
+ "afterTreeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "4af3f1f6eb137aed5e32146530a068601252760103e63e400f1ba89aadb5bd5e",
+ "size": 399,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "7df813f5c953407911e67c0d882560331cdde9c29bf4cc968340f278ebbc72e8",
+ "size": 399,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573904842-a233d4e4dc065.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 20.609917,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785573904842-a233d4e4dc065.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 15.667833,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [
+ {
+ "gradeVersion": "1.0",
+ "gradeId": "grade:a6f4b3fc7af28f082c443a5a75990ef242790b295e5de8fd0c38de92149221f1",
+ "graderId": "agent-field-update-grader",
+ "taskId": "sha256:618930ecee265100be229a2d2efa73e733f70c7131e00b0ed7a4bbb8ba75a18f",
+ "treeSha256": "sha256:f3b2df0c863b76dd11986b60ba95f484f23b791b9e6b191efb87c353ec0ca6f7",
+ "verdict": "passed",
+ "score": 3,
+ "maximumScore": 3,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-Ow65e1/workspace",
+ "field-update"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 33.318292,
+ "stdoutSha256": "sha256:fb2d6d6ccc2c7839e527f75052fa856b3e0b5c9aff34f424bd69337e5380f1c0",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "field-update: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ }
+ ],
+ "summary": {
+ "total": 2,
+ "passed": 2,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+ }
+ ],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 277,
+ "sha256": "sha256:8c5deb62c08c076632076d2296e1390fb414f0e18edf78255f777d1903671adc"
+ },
+ {
+ "id": "grade-task-grader",
+ "path": "grade-task-grader.json",
+ "mediaType": "application/json",
+ "bytes": 1932,
+ "sha256": "sha256:c7fee02d646c840e756f8dd0941d1f5aad23e05c9d5bb94cabdf3a4a212bfbbe"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 3227,
+ "sha256": "sha256:8472a72ad7f70be9e7486bbce2118e48ce8a74be6adae0ff130a69438d4395a9"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 187,
+ "sha256": "sha256:21f3f361c74972d195f556886659d9f83a5f002e17ea3ad0dd8bc004a46d7e62"
+ }
+ ],
+ "verdict": "passed",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 0,
+ "semanticVerdict": "passed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/TASK.md b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/TASK.md
new file mode 100644
index 0000000..ea3c5f0
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/TASK.md
@@ -0,0 +1 @@
+Extend the Pattern C card gallery with a third card. In lib/data/data.json add entity gamma with kind card, title Gamma Card, and color #8b5cf6; add alias Gamma Card -> gamma; append gamma to the gallery stage entities and domain.cardOrder. Update lib/data/data.js so SG_DATA_PACK exactly mirrors data.json. Do not modify the engine or HTML. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/agent.patch b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/agent.patch
new file mode 100644
index 0000000..fdba129
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/agent.patch
@@ -0,0 +1,49 @@
+diff --git a/lib/data/data.js b/lib/data/data.js
+--- a/lib/data/data.js
++++ b/lib/data/data.js
+@@ -1,1 +1,1 @@
+-globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta"]}],"domain":{"cardOrder":["alpha","beta"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
++globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"},"gamma":{"kind":"card","title":"Gamma Card","color":"#8b5cf6"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta","Gamma Card":"gamma"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta","gamma"]}],"domain":{"cardOrder":["alpha","beta","gamma"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -16,6 +16,11 @@
+ "beta": {
+ "kind": "card",
+ "title": "Beta Card",
+ "color": "#059669"
+- }
++ },
++ "gamma": {
++ "kind": "card",
++ "title": "Gamma Card",
++ "color": "#8b5cf6"
+ }
+ },
+ "aliases": {
+@@ -22,4 +27,5 @@
+ "aliases": {
+ "Alpha Card": "alpha",
+- "Beta Card": "beta"
++ "Beta Card": "beta",
++ "Gamma Card": "gamma"
+ },
+ "stages": [
+ {
+@@ -30,4 +36,5 @@
+ "entities": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
+ }
+ ],
+@@ -37,4 +44,5 @@
+ "cardOrder": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
+ },
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/patch-audit.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/patch-audit.json
new file mode 100644
index 0000000..78bc7bf
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/patch-audit.json
@@ -0,0 +1,99 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:7d65facd435834371e9a0a14897f625ff8b9b3c4d0e51f464cf0ae3d016c659b",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "rejected",
+ "patchSha256": "sha256:770dcba500dd445b1ee43e2717686d3ef2fbd57756a26d347f9c4104a2b991c3",
+ "patchBytes": 2461,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 4,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574435828-e4a2bdeb6ab5a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-T3JOxv/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 21.056834,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:8eb5fb3f1054381643c3673f6e5fa294934f64c17fbd0817aab44c6b4aacb205",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574435828-e4a2bdeb6ab5a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-T3JOxv/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 21.056834,
+ "duration": 21.056834,
+ "stdout": "",
+ "stderr": "error: corrupt patch at line 24\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/task-run.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/task-run.json
new file mode 100644
index 0000000..96d3fee
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r01/task-run.json
@@ -0,0 +1,223 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:d7c57685e7c52ac0c0cf5b5479eacc353a887873f09b066aea87e26baf9c5495",
+ "task": {
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "taskFileSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7",
+ "sourceRevision": "pattern-c-runtime-fixture-v1",
+ "contractTreeSha256": "sha256:0e1e4d837256995597bcf684eb3a2c613df90918e21bfdf7e8f3998a8f5e554c",
+ "runnerSnapshotBeforeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "runnerSnapshotFinalSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "pattern-c-runtime-r01"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-T3JOxv/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r01/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r01/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-T3JOxv/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 231543.67,
+ "stdoutSha256": "sha256:6c00198a6a0ef0c816deb24640f3124967218fb8d682e8cef6d6b1a8f2d9de09",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":2443,\"outputTokens\":19966,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.511429,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":2443,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":19966,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.511429,\"durationApiMs\":231215,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2443,
+ "outputTokens": 19966,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.511429,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2443,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 19966,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.511429,
+ "durationApiMs": 231215,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:7d65facd435834371e9a0a14897f625ff8b9b3c4d0e51f464cf0ae3d016c659b",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "rejected",
+ "patchSha256": "sha256:770dcba500dd445b1ee43e2717686d3ef2fbd57756a26d347f9c4104a2b991c3",
+ "patchBytes": 2461,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 4,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574435828-e4a2bdeb6ab5a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-T3JOxv/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 21.056834,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:8eb5fb3f1054381643c3673f6e5fa294934f64c17fbd0817aab44c6b4aacb205",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574435828-e4a2bdeb6ab5a.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-T3JOxv/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 21.056834,
+ "duration": 21.056834,
+ "stdout": "",
+ "stderr": "error: corrupt patch at line 24\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 2461,
+ "sha256": "sha256:770dcba500dd445b1ee43e2717686d3ef2fbd57756a26d347f9c4104a2b991c3"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 2915,
+ "sha256": "sha256:0cb4a082fe4af344ddebcb1253adadc46b85f545bbb4a60b1e882d97140bdefc"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 368,
+ "sha256": "sha256:8d6d410df008eb20405838885369f435267b07b38267305d013cd4661c971d29"
+ }
+ ],
+ "verdict": "patch-invalid",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 1,
+ "semanticVerdict": "patch-invalid",
+ "message": "git apply --check rejected the patch"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r02/TASK.md b/research/agent-eval/results/claude/trials/pattern-c-runtime-r02/TASK.md
new file mode 100644
index 0000000..ea3c5f0
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r02/TASK.md
@@ -0,0 +1 @@
+Extend the Pattern C card gallery with a third card. In lib/data/data.json add entity gamma with kind card, title Gamma Card, and color #8b5cf6; add alias Gamma Card -> gamma; append gamma to the gallery stage entities and domain.cardOrder. Update lib/data/data.js so SG_DATA_PACK exactly mirrors data.json. Do not modify the engine or HTML. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r02/task-run.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r02/task-run.json
new file mode 100644
index 0000000..5d4f237
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r02/task-run.json
@@ -0,0 +1,74 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:1e758add780932a28d4a8805f1027723d18ea577ceef2b37205920c827061b04",
+ "task": {
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "taskFileSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7",
+ "sourceRevision": "pattern-c-runtime-fixture-v1",
+ "contractTreeSha256": "sha256:0e1e4d837256995597bcf684eb3a2c613df90918e21bfdf7e8f3998a8f5e554c",
+ "runnerSnapshotBeforeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "runnerSnapshotFinalSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "pattern-c-runtime-r02"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-M2v46R/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r02/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r02/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-M2v46R/workspace",
+ "exitCode": 3,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 300049.4625,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:84c3787d7bf666ffee1ef31f289ba12d19f9551653d0082863d107d4b37668f9",
+ "stdout": "",
+ "stderr": "Error: spawnSync claude ETIMEDOUT\n at Object.spawnSync (node:internal/child_process:1120:20)\n at spawnSync (node:child_process:902:24)\n at Object. (/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js:41:18)\n at Module._compile (node:internal/modules/cjs/loader:1781:14)\n at Object..js (node:internal/modules/cjs/loader:1913:10)\n at Module.load (node:internal/modules/cjs/loader:1505:32)\n at Function._load (node:internal/modules/cjs/loader:1309:12)\n at wrapModuleLoad (node:internal/modules/cjs/loader:254:19)\n at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:171:5)\n at node:internal/main/run_main_module:36:49\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": null
+ },
+ "patch": null,
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 368,
+ "sha256": "sha256:8d6d410df008eb20405838885369f435267b07b38267305d013cd4661c971d29"
+ }
+ ],
+ "verdict": "infra-error",
+ "termination": {
+ "kind": "infrastructure-error",
+ "processExitCode": 3,
+ "semanticVerdict": "infra-error",
+ "message": "agent command failed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/TASK.md b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/TASK.md
new file mode 100644
index 0000000..ea3c5f0
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/TASK.md
@@ -0,0 +1 @@
+Extend the Pattern C card gallery with a third card. In lib/data/data.json add entity gamma with kind card, title Gamma Card, and color #8b5cf6; add alias Gamma Card -> gamma; append gamma to the gallery stage entities and domain.cardOrder. Update lib/data/data.js so SG_DATA_PACK exactly mirrors data.json. Do not modify the engine or HTML. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/agent.patch b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/agent.patch
new file mode 100644
index 0000000..ecb0051
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/agent.patch
@@ -0,0 +1,44 @@
+diff --git a/lib/data/data.js b/lib/data/data.js
+--- a/lib/data/data.js
++++ b/lib/data/data.js
+@@ -1,1 +1,1 @@
+-globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta"]}],"domain":{"cardOrder":["alpha","beta"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
++globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"},"gamma":{"kind":"card","title":"Gamma Card","color":"#8b5cf6"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta","Gamma Card":"gamma"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta","gamma"]}],"domain":{"cardOrder":["alpha","beta","gamma"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -17,7 +17,12 @@
+ "kind": "card",
+ "title": "Beta Card",
+ "color": "#059669"
+- }
++ },
++ "gamma": {
++ "kind": "card",
++ "title": "Gamma Card",
++ "color": "#8b5cf6"
++ }
+ },
+ "aliases": {
+ "Alpha Card": "alpha",
+@@ -22,4 +22,5 @@
+ "aliases": {
+ "Alpha Card": "alpha",
+- "Beta Card": "beta"
++ "Beta Card": "beta",
++ "Gamma Card": "gamma"
+ },
+@@ -30,4 +30,5 @@
+ "entities": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
+@@ -37,4 +37,5 @@
+ "cardOrder": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/patch-audit.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/patch-audit.json
new file mode 100644
index 0000000..24ceb3b
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/patch-audit.json
@@ -0,0 +1,99 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:91cac30579863ebb581b53be329ced063ca6ebcfb49a3205e1823d7779b788a4",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "rejected",
+ "patchSha256": "sha256:00f856066a26111dbb0dc23afa0d89710e9d54a1cd69a43cbb8d138d7e583f6e",
+ "patchBytes": 2433,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 4,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574836572-a1cd82ff18c64.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-8VTQr7/workspace",
+ "exitCode": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 19.206875,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:054b15b4694a5d12e4cb51eb6b5d31e7b015408f2415eafe5e3a654029592696",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574836572-a1cd82ff18c64.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-8VTQr7/workspace",
+ "status": 1,
+ "exitCode": 1,
+ "exit": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 19.206875,
+ "duration": 19.206875,
+ "stdout": "",
+ "stderr": "error: patch failed: lib/data/data.json:22\nerror: lib/data/data.json: patch does not apply\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/task-run.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/task-run.json
new file mode 100644
index 0000000..9a86b04
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r03/task-run.json
@@ -0,0 +1,223 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:495af23afb9d35b3d74f0547b57ef22f80e85b980877159bc46e3e6db566f3f1",
+ "task": {
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "taskFileSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7",
+ "sourceRevision": "pattern-c-runtime-fixture-v1",
+ "contractTreeSha256": "sha256:0e1e4d837256995597bcf684eb3a2c613df90918e21bfdf7e8f3998a8f5e554c",
+ "runnerSnapshotBeforeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "runnerSnapshotFinalSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "pattern-c-runtime-r03"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-8VTQr7/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r03/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r03/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-8VTQr7/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 100588.355334,
+ "stdoutSha256": "sha256:2ae23305d145989abcb4da3b58b64a9b58e442964f372afb92af0ea32d233a00",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":2444,\"outputTokens\":9733,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.25560900000000003,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":2444,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":9733,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.25560900000000003,\"durationApiMs\":100259,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2444,
+ "outputTokens": 9733,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.25560900000000003,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2444,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 9733,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.25560900000000003,
+ "durationApiMs": 100259,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:91cac30579863ebb581b53be329ced063ca6ebcfb49a3205e1823d7779b788a4",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "rejected",
+ "patchSha256": "sha256:00f856066a26111dbb0dc23afa0d89710e9d54a1cd69a43cbb8d138d7e583f6e",
+ "patchBytes": 2433,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 4,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574836572-a1cd82ff18c64.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-8VTQr7/workspace",
+ "exitCode": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 19.206875,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:054b15b4694a5d12e4cb51eb6b5d31e7b015408f2415eafe5e3a654029592696",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574836572-a1cd82ff18c64.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-8VTQr7/workspace",
+ "status": 1,
+ "exitCode": 1,
+ "exit": 1,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 19.206875,
+ "duration": 19.206875,
+ "stdout": "",
+ "stderr": "error: patch failed: lib/data/data.json:22\nerror: lib/data/data.json: patch does not apply\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 2433,
+ "sha256": "sha256:00f856066a26111dbb0dc23afa0d89710e9d54a1cd69a43cbb8d138d7e583f6e"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 2967,
+ "sha256": "sha256:7127ed9c936434ee71a1eebddf449b10281b89292f11b389c283c06e309abef0"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 368,
+ "sha256": "sha256:8d6d410df008eb20405838885369f435267b07b38267305d013cd4661c971d29"
+ }
+ ],
+ "verdict": "patch-invalid",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 1,
+ "semanticVerdict": "patch-invalid",
+ "message": "git apply --check rejected the patch"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/TASK.md b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/TASK.md
new file mode 100644
index 0000000..ea3c5f0
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/TASK.md
@@ -0,0 +1 @@
+Extend the Pattern C card gallery with a third card. In lib/data/data.json add entity gamma with kind card, title Gamma Card, and color #8b5cf6; add alias Gamma Card -> gamma; append gamma to the gallery stage entities and domain.cardOrder. Update lib/data/data.js so SG_DATA_PACK exactly mirrors data.json. Do not modify the engine or HTML. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/agent.patch b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/agent.patch
new file mode 100644
index 0000000..f51759e
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/agent.patch
@@ -0,0 +1,50 @@
+diff --git a/lib/data/data.js b/lib/data/data.js
+--- a/lib/data/data.js
++++ b/lib/data/data.js
+@@ -1,1 +1,1 @@
+-globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta"]}],"domain":{"cardOrder":["alpha","beta"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
++globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"},"gamma":{"kind":"card","title":"Gamma Card","color":"#8b5cf6"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta","Gamma Card":"gamma"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta","gamma"]}],"domain":{"cardOrder":["alpha","beta","gamma"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -14,27 +14,35 @@
+ "color": "#2563eb"
+ },
+ "beta": {
+ "kind": "card",
+ "title": "Beta Card",
+ "color": "#059669"
+- }
++ },
++ "gamma": {
++ "kind": "card",
++ "title": "Gamma Card",
++ "color": "#8b5cf6"
++ }
+ },
+ "aliases": {
+ "Alpha Card": "alpha",
+- "Beta Card": "beta"
++ "Beta Card": "beta",
++ "Gamma Card": "gamma"
+ },
+ "stages": [
+ {
+ "key": "gallery",
+ "name": "Gallery",
+ "entities": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
+ }
+ ],
+ "domain": {
+ "cardOrder": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
+ },
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate-stability.dom.html b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate-stability.dom.html
new file mode 100644
index 0000000..d707565
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate-stability.dom.html
@@ -0,0 +1,9 @@
+
+Alpha Card Beta Card Gamma Card
+
+
+
+
+
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate-stability.png b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate-stability.png
new file mode 100644
index 0000000..682cc3c
Binary files /dev/null and b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate-stability.png differ
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate.dom.html b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate.dom.html
new file mode 100644
index 0000000..d707565
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate.dom.html
@@ -0,0 +1,9 @@
+
+Alpha Card Beta Card Gamma Card
+
+
+
+
+
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate.png b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate.png
new file mode 100644
index 0000000..682cc3c
Binary files /dev/null and b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/candidate.png differ
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/grade-task-grader.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/grade-task-grader.json
new file mode 100644
index 0000000..a7a64c4
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/grade-task-grader.json
@@ -0,0 +1,153 @@
+{
+ "gradeVersion": "1.0",
+ "gradeId": "grade:c78beeeac5f60005832e9564d1a61dde325ec209d89872bb463dd53faa673e7e",
+ "graderId": "agent-pattern-c-grader",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "treeSha256": "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "verdict": "passed",
+ "score": 8,
+ "maximumScore": 8,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "pattern-c"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 29.076083,
+ "stdoutSha256": "sha256:14aa7fc7ee2174870b13c88f2eefc39873ff15b29013e54803fc3b21cf099c88",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "pattern-c: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ },
+ {
+ "id": "browser-producer",
+ "type": "command",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/trusted-graders/task-grader/browser-evidence.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r04",
+ "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/trusted-tools"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 13661.467541,
+ "stdoutSha256": "sha256:158a8b1e9f61502cf43484af3ae2c816dbc0acc8e44b755e05d9e1cd50896553",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"runtime\":\"runtime:3cf2716ed67e4776e7fb5a1eb7ba9dcf68df327a1163d112a1defd0509c1d993\",\"visual\":\"visual:b3d9ffed355ba9dda006f8c7b3962445098fec0479e09b6a39ddfde0e64a010f\",\"pixelDiffRatio\":0,\"stabilityDiffRatio\":0}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ },
+ {
+ "id": "runtime-mount",
+ "type": "runtime-evidence",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "scenarioId": "pattern-c-gallery",
+ "file": "runtime-evidence.json",
+ "evidenceId": "runtime:3cf2716ed67e4776e7fb5a1eb7ba9dcf68df327a1163d112a1defd0509c1d993",
+ "validation": {
+ "valid": true,
+ "errors": [],
+ "status": "passed",
+ "summary": {
+ "assertions": 4,
+ "passed": 4,
+ "failed": 0
+ }
+ }
+ }
+ },
+ {
+ "id": "visual-regression",
+ "type": "visual-evidence",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "scenarioId": "pattern-c-gallery-desktop",
+ "file": "visual-evidence.json",
+ "evidenceId": "visual:b3d9ffed355ba9dda006f8c7b3962445098fec0479e09b6a39ddfde0e64a010f",
+ "validation": {
+ "valid": true,
+ "errors": [],
+ "status": "passed",
+ "scenarios": [
+ {
+ "id": "pattern-c-gallery-desktop",
+ "status": "passed",
+ "errors": []
+ }
+ ],
+ "summary": {
+ "total": 1,
+ "passed": 1,
+ "failed": 0,
+ "notAssessed": 0
+ }
+ }
+ }
+ }
+ ],
+ "summary": {
+ "total": 5,
+ "passed": 5,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/patch-audit.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/patch-audit.json
new file mode 100644
index 0000000..1ecc717
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/patch-audit.json
@@ -0,0 +1,138 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:71797791c1925890fd330ecbb26f2d483cf7a0e34a103fc8a6867a4039510185",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "applied",
+ "patchSha256": "sha256:fbd1f3cf9582212db5f48c34217ddf4a7fd31144cce8cd8659daf512ba67636d",
+ "patchBytes": 2492,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.js",
+ "relativePath": "lib/data/data.js",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "088f58c2e5beb40627f2c6f5c8439d13166593fe055b92e8592d1497561d73e9",
+ "size": 743,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.js",
+ "relativePath": "lib/data/data.js",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "e286223efbf752e0acffdb10ff5c18ecd0853831b44762cf39c1a7bbbd5c511c",
+ "size": 843,
+ "target": null
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "c63bb685fd7d6ebdb624eee3a04edd4258fe7ba5ab7594b3302cc05adc29d814",
+ "size": 1079,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "44a4a32b20a0795ec3bde9ee5a5464bb951b34fa49db0184bbc73ef6fe01b094",
+ "size": 1236,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574963625-2400b6c2df569.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 34.301417,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574963625-2400b6c2df569.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 15.816708,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/reference.dom.html b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/reference.dom.html
new file mode 100644
index 0000000..d707565
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/reference.dom.html
@@ -0,0 +1,9 @@
+
+Alpha Card Beta Card Gamma Card
+
+
+
+
+
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/reference.png b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/reference.png
new file mode 100644
index 0000000..682cc3c
Binary files /dev/null and b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/reference.png differ
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/runtime-evidence.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/runtime-evidence.json
new file mode 100644
index 0000000..0fd4b9a
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/runtime-evidence.json
@@ -0,0 +1,54 @@
+{
+ "evidenceVersion": "1.0",
+ "kind": "runtime-evidence",
+ "evidenceId": "runtime:3cf2716ed67e4776e7fb5a1eb7ba9dcf68df327a1163d112a1defd0509c1d993",
+ "scenarioId": "pattern-c-gallery",
+ "subjectTreeSha256": "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "producer": {
+ "name": "headless-chrome-pattern-c",
+ "version": "1.2.0"
+ },
+ "environment": {
+ "browser": "Google Chrome 151.0.7922.71",
+ "platform": "darwin",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670"
+ },
+ "assertions": [
+ {
+ "id": "error-collector",
+ "status": "passed"
+ },
+ {
+ "id": "mount-status",
+ "status": "passed",
+ "actual": "passed"
+ },
+ {
+ "id": "card-count",
+ "status": "passed",
+ "actual": 3,
+ "expected": 3
+ },
+ {
+ "id": "gamma-visible",
+ "status": "passed"
+ }
+ ],
+ "consoleErrors": [],
+ "pageErrors": [],
+ "networkFailures": [],
+ "artifacts": [
+ {
+ "id": "candidate-dom",
+ "path": "candidate.dom.html",
+ "mediaType": "text/html",
+ "sha256": "sha256:19aeed698f2408742b66ca6d4cc7b63ef9dd6abe55458a7e55d49b95c5c1d9a3"
+ },
+ {
+ "id": "candidate-browser-log",
+ "path": "candidate.chrome.log",
+ "mediaType": "text/plain",
+ "sha256": "sha256:8d2174e4152b3d97fe4e6008b008f23a0eebd64c42b7599bba791ff61cc35e54"
+ }
+ ]
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/stability-diff.png b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/stability-diff.png
new file mode 100644
index 0000000..cfd7137
Binary files /dev/null and b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/stability-diff.png differ
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/task-run.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/task-run.json
new file mode 100644
index 0000000..3addcb4
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/task-run.json
@@ -0,0 +1,513 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:d760c86dff6d4d960aca95fe2473ce10baaa537bc94534635abc413a1aab1310",
+ "task": {
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "taskFileSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7",
+ "sourceRevision": "pattern-c-runtime-fixture-v1",
+ "contractTreeSha256": "sha256:0e1e4d837256995597bcf684eb3a2c613df90918e21bfdf7e8f3998a8f5e554c",
+ "runnerSnapshotBeforeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "runnerSnapshotFinalSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "pattern-c-runtime-r04"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r04/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r04/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 126989.818792,
+ "stdoutSha256": "sha256:7306018546f493d1aa3f24f637eaa3bf2b6a9eb5054d0de37bc2270229c68074",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":2443,\"outputTokens\":11854,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.308629,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":2443,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":11854,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.308629,\"durationApiMs\":126661,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2443,
+ "outputTokens": 11854,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.308629,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2443,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 11854,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.308629,
+ "durationApiMs": 126661,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:71797791c1925890fd330ecbb26f2d483cf7a0e34a103fc8a6867a4039510185",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "applied",
+ "patchSha256": "sha256:fbd1f3cf9582212db5f48c34217ddf4a7fd31144cce8cd8659daf512ba67636d",
+ "patchBytes": 2492,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ }
+ ],
+ "actualOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.js",
+ "relativePath": "lib/data/data.js",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "088f58c2e5beb40627f2c6f5c8439d13166593fe055b92e8592d1497561d73e9",
+ "size": 743,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.js",
+ "relativePath": "lib/data/data.js",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "e286223efbf752e0acffdb10ff5c18ecd0853831b44762cf39c1a7bbbd5c511c",
+ "size": 843,
+ "target": null
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "allowed": true,
+ "reason": null,
+ "before": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "c63bb685fd7d6ebdb624eee3a04edd4258fe7ba5ab7594b3302cc05adc29d814",
+ "size": 1079,
+ "target": null
+ },
+ "after": {
+ "path": "lib/data/data.json",
+ "relativePath": "lib/data/data.json",
+ "kind": "file",
+ "mode": 420,
+ "sha256": "44a4a32b20a0795ec3bde9ee5a5464bb951b34fa49db0184bbc73ef6fe01b094",
+ "size": 1236,
+ "target": null
+ }
+ }
+ ],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574963625-2400b6c2df569.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 34.301417,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ },
+ "apply": {
+ "argv": [
+ "git",
+ "apply",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785574963625-2400b6c2df569.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 15.816708,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "error": null
+ }
+ },
+ "failure": null,
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [
+ {
+ "gradeVersion": "1.0",
+ "gradeId": "grade:c78beeeac5f60005832e9564d1a61dde325ec209d89872bb463dd53faa673e7e",
+ "graderId": "agent-pattern-c-grader",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "treeSha256": "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "verdict": "passed",
+ "score": 8,
+ "maximumScore": 8,
+ "results": [
+ {
+ "id": "data-pack-contract",
+ "type": "data-pack-contract",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "errors": [],
+ "warnings": [],
+ "strict": true
+ }
+ },
+ {
+ "id": "hidden-property",
+ "type": "command",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/trusted-graders/task-grader/assert-task.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "pattern-c"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 29.076083,
+ "stdoutSha256": "sha256:14aa7fc7ee2174870b13c88f2eefc39873ff15b29013e54803fc3b21cf099c88",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "pattern-c: complete hidden candidate passed\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ },
+ {
+ "id": "browser-producer",
+ "type": "command",
+ "required": true,
+ "weight": 1,
+ "status": "passed",
+ "score": 1,
+ "findings": [],
+ "evidence": {
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/trusted-graders/task-grader/browser-evidence.js",
+ "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r04",
+ "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-O4TaVb/trusted-tools"
+ ],
+ "cwd": ".",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 13661.467541,
+ "stdoutSha256": "sha256:158a8b1e9f61502cf43484af3ae2c816dbc0acc8e44b755e05d9e1cd50896553",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"runtime\":\"runtime:3cf2716ed67e4776e7fb5a1eb7ba9dcf68df327a1163d112a1defd0509c1d993\",\"visual\":\"visual:b3d9ffed355ba9dda006f8c7b3962445098fec0479e09b6a39ddfde0e64a010f\",\"pixelDiffRatio\":0,\"stabilityDiffRatio\":0}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null
+ }
+ },
+ {
+ "id": "runtime-mount",
+ "type": "runtime-evidence",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "scenarioId": "pattern-c-gallery",
+ "file": "runtime-evidence.json",
+ "evidenceId": "runtime:3cf2716ed67e4776e7fb5a1eb7ba9dcf68df327a1163d112a1defd0509c1d993",
+ "validation": {
+ "valid": true,
+ "errors": [],
+ "status": "passed",
+ "summary": {
+ "assertions": 4,
+ "passed": 4,
+ "failed": 0
+ }
+ }
+ }
+ },
+ {
+ "id": "visual-regression",
+ "type": "visual-evidence",
+ "required": true,
+ "weight": 2,
+ "status": "passed",
+ "score": 2,
+ "findings": [],
+ "evidence": {
+ "scenarioId": "pattern-c-gallery-desktop",
+ "file": "visual-evidence.json",
+ "evidenceId": "visual:b3d9ffed355ba9dda006f8c7b3962445098fec0479e09b6a39ddfde0e64a010f",
+ "validation": {
+ "valid": true,
+ "errors": [],
+ "status": "passed",
+ "scenarios": [
+ {
+ "id": "pattern-c-gallery-desktop",
+ "status": "passed",
+ "errors": []
+ }
+ ],
+ "summary": {
+ "total": 1,
+ "passed": 1,
+ "failed": 0,
+ "notAssessed": 0
+ }
+ }
+ }
+ }
+ ],
+ "summary": {
+ "total": 5,
+ "passed": 5,
+ "failed": 0,
+ "notAssessed": 0,
+ "invalid": 0,
+ "infraErrors": 0,
+ "requiredIncomplete": 0
+ },
+ "manifestWeight": 1
+ }
+ ],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 2492,
+ "sha256": "sha256:fbd1f3cf9582212db5f48c34217ddf4a7fd31144cce8cd8659daf512ba67636d"
+ },
+ {
+ "id": "generated-candidate-stability.chrome.log",
+ "path": "candidate-stability.chrome.log",
+ "mediaType": "text/plain",
+ "bytes": 552,
+ "sha256": "sha256:d434687df41aa7918b3c7a5a0e54e7f4620f864ecb40b817344a12a52e48c98d"
+ },
+ {
+ "id": "generated-candidate-stability.dom.html",
+ "path": "candidate-stability.dom.html",
+ "mediaType": "text/html",
+ "bytes": 2108,
+ "sha256": "sha256:19aeed698f2408742b66ca6d4cc7b63ef9dd6abe55458a7e55d49b95c5c1d9a3"
+ },
+ {
+ "id": "generated-candidate-stability.png",
+ "path": "candidate-stability.png",
+ "mediaType": "image/png",
+ "bytes": 91725,
+ "sha256": "sha256:369ad75f907e2fea0a06e5182a03c3d1d4f6570b531958dabf10396c2e2cd3ea"
+ },
+ {
+ "id": "generated-candidate.chrome.log",
+ "path": "candidate.chrome.log",
+ "mediaType": "text/plain",
+ "bytes": 1983,
+ "sha256": "sha256:8d2174e4152b3d97fe4e6008b008f23a0eebd64c42b7599bba791ff61cc35e54"
+ },
+ {
+ "id": "generated-candidate.dom.html",
+ "path": "candidate.dom.html",
+ "mediaType": "text/html",
+ "bytes": 2108,
+ "sha256": "sha256:19aeed698f2408742b66ca6d4cc7b63ef9dd6abe55458a7e55d49b95c5c1d9a3"
+ },
+ {
+ "id": "generated-candidate.png",
+ "path": "candidate.png",
+ "mediaType": "image/png",
+ "bytes": 91725,
+ "sha256": "sha256:369ad75f907e2fea0a06e5182a03c3d1d4f6570b531958dabf10396c2e2cd3ea"
+ },
+ {
+ "id": "grade-task-grader",
+ "path": "grade-task-grader.json",
+ "mediaType": "application/json",
+ "bytes": 4934,
+ "sha256": "sha256:add2a1269eaea164680339d1b800b5cd70ecd25794bc15b3e58653d5caac58af"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 4167,
+ "sha256": "sha256:883a11b398c21227df3a88ba2839f13c8670cbcda5f190a84f33b66443246dde"
+ },
+ {
+ "id": "generated-reference.chrome.log",
+ "path": "reference.chrome.log",
+ "mediaType": "text/plain",
+ "bytes": 255,
+ "sha256": "sha256:c89dec168c7f15c1ddbf83a170a0bdc525a15edb97d943a75a519f28d0829831"
+ },
+ {
+ "id": "generated-reference.dom.html",
+ "path": "reference.dom.html",
+ "mediaType": "text/html",
+ "bytes": 2108,
+ "sha256": "sha256:19aeed698f2408742b66ca6d4cc7b63ef9dd6abe55458a7e55d49b95c5c1d9a3"
+ },
+ {
+ "id": "generated-reference.png",
+ "path": "reference.png",
+ "mediaType": "image/png",
+ "bytes": 91725,
+ "sha256": "sha256:369ad75f907e2fea0a06e5182a03c3d1d4f6570b531958dabf10396c2e2cd3ea"
+ },
+ {
+ "id": "generated-runtime-evidence.json",
+ "path": "runtime-evidence.json",
+ "mediaType": "application/json",
+ "bytes": 1411,
+ "sha256": "sha256:7706dd7d5df7cc1e03858040aca532e0db5b537f90cb7008d1ee7cc25291c948"
+ },
+ {
+ "id": "generated-stability-diff.png",
+ "path": "stability-diff.png",
+ "mediaType": "image/png",
+ "bytes": 3150,
+ "sha256": "sha256:aef339735a9bff865bfe7912b73ab3c2ee6f41c2aa4482ed0cf006381a82b775"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 368,
+ "sha256": "sha256:8d6d410df008eb20405838885369f435267b07b38267305d013cd4661c971d29"
+ },
+ {
+ "id": "generated-visual-diff.png",
+ "path": "visual-diff.png",
+ "mediaType": "image/png",
+ "bytes": 3150,
+ "sha256": "sha256:aef339735a9bff865bfe7912b73ab3c2ee6f41c2aa4482ed0cf006381a82b775"
+ },
+ {
+ "id": "generated-visual-evidence.json",
+ "path": "visual-evidence.json",
+ "mediaType": "application/json",
+ "bytes": 2101,
+ "sha256": "sha256:4a873f05810c949bd2357d3d90872e5238a9a49f2424d35d15858e4faecdf902"
+ }
+ ],
+ "verdict": "passed",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 0,
+ "semanticVerdict": "passed"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/visual-diff.png b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/visual-diff.png
new file mode 100644
index 0000000..cfd7137
Binary files /dev/null and b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/visual-diff.png differ
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/visual-evidence.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/visual-evidence.json
new file mode 100644
index 0000000..a6bee82
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r04/visual-evidence.json
@@ -0,0 +1,65 @@
+{
+ "evidenceVersion": "1.0",
+ "kind": "visual-evidence",
+ "evidenceId": "visual:b3d9ffed355ba9dda006f8c7b3962445098fec0479e09b6a39ddfde0e64a010f",
+ "referenceTreeSha256": "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "candidateTreeSha256": "sha256:b48495ceef97ecc05c4a6f85ab102ff5a3858110275202b3fae365252af113db",
+ "configurationSha256": "sha256:a6a063638a9b86f8d9c89b8dede46bc2069caf1067400f293fdf3ff6c185a41e",
+ "producer": {
+ "name": "headless-chrome-pillow",
+ "version": "1.2.0"
+ },
+ "thresholds": {
+ "maxPixelDiffRatio": 0.02,
+ "minComputedStyleScore": 0.98,
+ "minCoverage": 1
+ },
+ "scenarios": [
+ {
+ "id": "pattern-c-gallery-desktop",
+ "viewport": {
+ "width": 1100,
+ "height": 720
+ },
+ "pixelDiffRatio": 0,
+ "computedStyleScore": 1,
+ "coverage": 1,
+ "stabilityFailures": 0,
+ "referenceArtifact": "reference-screenshot",
+ "candidateArtifact": "candidate-screenshot",
+ "diffArtifact": "diff-screenshot"
+ }
+ ],
+ "artifacts": [
+ {
+ "id": "reference-screenshot",
+ "path": "reference.png",
+ "mediaType": "image/png",
+ "sha256": "sha256:369ad75f907e2fea0a06e5182a03c3d1d4f6570b531958dabf10396c2e2cd3ea"
+ },
+ {
+ "id": "candidate-screenshot",
+ "path": "candidate.png",
+ "mediaType": "image/png",
+ "sha256": "sha256:369ad75f907e2fea0a06e5182a03c3d1d4f6570b531958dabf10396c2e2cd3ea"
+ },
+ {
+ "id": "diff-screenshot",
+ "path": "visual-diff.png",
+ "mediaType": "image/png",
+ "sha256": "sha256:aef339735a9bff865bfe7912b73ab3c2ee6f41c2aa4482ed0cf006381a82b775"
+ },
+ {
+ "id": "candidate-stability-screenshot",
+ "path": "candidate-stability.png",
+ "mediaType": "image/png",
+ "sha256": "sha256:369ad75f907e2fea0a06e5182a03c3d1d4f6570b531958dabf10396c2e2cd3ea"
+ },
+ {
+ "id": "stability-diff-screenshot",
+ "path": "stability-diff.png",
+ "mediaType": "image/png",
+ "sha256": "sha256:aef339735a9bff865bfe7912b73ab3c2ee6f41c2aa4482ed0cf006381a82b775"
+ }
+ ]
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/TASK.md b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/TASK.md
new file mode 100644
index 0000000..ea3c5f0
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/TASK.md
@@ -0,0 +1 @@
+Extend the Pattern C card gallery with a third card. In lib/data/data.json add entity gamma with kind card, title Gamma Card, and color #8b5cf6; add alias Gamma Card -> gamma; append gamma to the gallery stage entities and domain.cardOrder. Update lib/data/data.js so SG_DATA_PACK exactly mirrors data.json. Do not modify the engine or HTML. Emit a unified diff only.
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/agent.patch b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/agent.patch
new file mode 100644
index 0000000..caf6e45
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/agent.patch
@@ -0,0 +1,51 @@
+diff --git a/lib/data/data.json b/lib/data/data.json
+--- a/lib/data/data.json
++++ b/lib/data/data.json
+@@ -14,8 +14,13 @@
+ "color": "#2563eb"
+ },
+ "beta": {
+ "kind": "card",
+ "title": "Beta Card",
+ "color": "#059669"
+- }
++ },
++ "gamma": {
++ "kind": "card",
++ "title": "Gamma Card",
++ "color": "#8b5cf6"
++ }
+ },
+ "aliases": {
+@@ -22,5 +27,6 @@
+ "aliases": {
+ "Alpha Card": "alpha",
+- "Beta Card": "beta"
++ "Beta Card": "beta",
++ "Gamma Card": "gamma"
+ },
+ "stages": [
+@@ -28,6 +34,7 @@
+ "key": "gallery",
+ "name": "Gallery",
+ "entities": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
+@@ -36,6 +43,7 @@
+ "domain": {
+ "cardOrder": [
+ "alpha",
+- "beta"
++ "beta",
++ "gamma"
+ ]
+ },
+diff --git a/lib/data/data.js b/lib/data/data.js
+--- a/lib/data/data.js
++++ b/lib/data/data.js
+@@ -1 +1 @@
+-globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta"]}],"domain":{"cardOrder":["alpha","beta"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
++globalThis.SG_DATA_PACK = {"schemaVersion":"1.3","meta":{"id":"agent-pattern-c","title":"Pattern C runtime fixture"},"kindNameFields":{"card":"title"},"entities":{"alpha":{"kind":"card","title":"Alpha Card","color":"#2563eb"},"beta":{"kind":"card","title":"Beta Card","color":"#059669"},"gamma":{"kind":"card","title":"Gamma Card","color":"#8b5cf6"}},"aliases":{"Alpha Card":"alpha","Beta Card":"beta","Gamma Card":"gamma"},"stages":[{"key":"gallery","name":"Gallery","entities":["alpha","beta","gamma"]}],"domain":{"cardOrder":["alpha","beta","gamma"]},"derivations":{"cardGallery":{"kind":"projection","source":"domain.cardOrder","note":"Projects canonical card entities into the Pattern C gallery options shape.","alsoTouches":["entities.*.title","entities.*.color"],"consumers":["__fromPack","#mount .card"],"affects":["#mount .card"]}}};
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/patch-audit.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/patch-audit.json
new file mode 100644
index 0000000..0fd5870
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/patch-audit.json
@@ -0,0 +1,99 @@
+{
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:3596347f157a38d17316843de64d66c921dfb4a34e364eb2279d7dfa4baea45c",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "rejected",
+ "patchSha256": "sha256:2649706bf415a7b28cf09ce7075f9d6d3017a869cf66e8015c13dda437d174e8",
+ "patchBytes": 2537,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 4,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785575176925-f9951cf8b0d45.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-mzh4Hq/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 26.0165,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:62ccebd4d1a58360a0ab84d7032ef868076a6b813b79dd5f2292710f327e0bcb",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785575176925-f9951cf8b0d45.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-mzh4Hq/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 26.0165,
+ "duration": 26.0165,
+ "stdout": "",
+ "stderr": "error: patch fragment without header at line 20: @@ -22,5 +27,6 @@\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+}
diff --git a/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/task-run.json b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/task-run.json
new file mode 100644
index 0000000..330d1f2
--- /dev/null
+++ b/research/agent-eval/results/claude/trials/pattern-c-runtime-r05/task-run.json
@@ -0,0 +1,223 @@
+{
+ "runVersion": "1.0",
+ "runId": "task-run:07e9c45767c9629be80216750400fa6df478397e9d3bca19e06fe327616100a7",
+ "task": {
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "taskFileSha256": "sha256:9220e12596b83dbc1619f4859c54d2c3847b733b700818a50fb97aa88df3f8c7",
+ "sourceRevision": "pattern-c-runtime-fixture-v1",
+ "contractTreeSha256": "sha256:0e1e4d837256995597bcf684eb3a2c613df90918e21bfdf7e8f3998a8f5e554c",
+ "runnerSnapshotBeforeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "runnerSnapshotFinalSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "sourceUnchanged": true,
+ "sourceFinalError": null,
+ "trialId": "pattern-c-runtime-r05"
+ },
+ "agent": {
+ "provider": "claude-code-2.1.185",
+ "model": "sonnet alias (provider metadata recorded)",
+ "argv": [
+ "/Users/tangyaoyue/.hermes/node/bin/node",
+ "/Users/tangyaoyue/DEV/sg-data-pack/research/agent-eval/providers/claude-patch-agent.js",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-mzh4Hq/workspace",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r05/agent.patch",
+ "/private/tmp/sg-agent-claude-final-release/trials/pattern-c-runtime-r05/TASK.md"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-mzh4Hq/workspace",
+ "exitCode": 0,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 199414.87625,
+ "stdoutSha256": "sha256:77c5d7c684ddcaa577d51271c8e482339575c3081e15bda51cf3fe163ad3b4c5",
+ "stderrSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stdout": "{\"provider\":\"claude-code\",\"declaredModel\":\"sonnet\",\"modelUsage\":{\"MiniMax-M3[1M]\":{\"inputTokens\":2444,\"outputTokens\":16845,\"cacheReadInputTokens\":128,\"cacheCreationInputTokens\":0,\"webSearchRequests\":0,\"costUSD\":0.433409,\"contextWindow\":1000000,\"maxOutputTokens\":32000}},\"usage\":{\"input_tokens\":2444,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":128,\"output_tokens\":16845,\"server_tool_use\":{\"web_search_requests\":0,\"web_fetch_requests\":0},\"service_tier\":\"standard\",\"cache_creation\":{\"ephemeral_1h_input_tokens\":0,\"ephemeral_5m_input_tokens\":0},\"inference_geo\":\"\",\"iterations\":[],\"speed\":\"standard\"},\"totalCostUsd\":0.433409,\"durationApiMs\":198830,\"stopReason\":\"end_turn\",\"permissionDenials\":[]}\n",
+ "stderr": "",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "error": null,
+ "providerMetadata": {
+ "provider": "claude-code",
+ "declaredModel": "sonnet",
+ "modelUsage": {
+ "MiniMax-M3[1M]": {
+ "inputTokens": 2444,
+ "outputTokens": 16845,
+ "cacheReadInputTokens": 128,
+ "cacheCreationInputTokens": 0,
+ "webSearchRequests": 0,
+ "costUSD": 0.433409,
+ "contextWindow": 1000000,
+ "maxOutputTokens": 32000
+ }
+ },
+ "usage": {
+ "input_tokens": 2444,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 128,
+ "output_tokens": 16845,
+ "server_tool_use": {
+ "web_search_requests": 0,
+ "web_fetch_requests": 0
+ },
+ "service_tier": "standard",
+ "cache_creation": {
+ "ephemeral_1h_input_tokens": 0,
+ "ephemeral_5m_input_tokens": 0
+ },
+ "inference_geo": "",
+ "iterations": [],
+ "speed": "standard"
+ },
+ "totalCostUsd": 0.433409,
+ "durationApiMs": 198830,
+ "stopReason": "end_turn",
+ "permissionDenials": []
+ }
+ },
+ "patch": {
+ "auditVersion": "1.0",
+ "auditId": "patch-audit:3596347f157a38d17316843de64d66c921dfb4a34e364eb2279d7dfa4baea45c",
+ "kind": "patch-audit",
+ "taskId": "sha256:3c621ced615d86b4b1c9e8ea644b1045fd762b71675f5c82acc54737f9182670",
+ "status": "rejected",
+ "patchSha256": "sha256:2649706bf415a7b28cf09ce7075f9d6d3017a869cf66e8015c13dda437d174e8",
+ "patchBytes": 2537,
+ "beforeTreeSha256": "sha256:2f5a60bc694ee969c17914c85b53df4960565372eb274588e287a2715478b865",
+ "afterTreeSha256": null,
+ "declaredOperations": [
+ {
+ "operation": "modify",
+ "path": "lib/data/data.json",
+ "hunks": 4,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.json"
+ }
+ },
+ {
+ "operation": "modify",
+ "path": "lib/data/data.js",
+ "hunks": 1,
+ "modes": {},
+ "policy": {
+ "allowed": true,
+ "reason": null,
+ "deniedBy": null,
+ "allowedBy": "lib/data/data.js"
+ }
+ }
+ ],
+ "actualOperations": [],
+ "preflight": {
+ "allowed": true,
+ "violations": []
+ },
+ "postflight": null,
+ "commands": {
+ "check": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785575176925-f9951cf8b0d45.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-mzh4Hq/workspace",
+ "exitCode": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 26.0165,
+ "stdoutSha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "stderrSha256": "sha256:62ccebd4d1a58360a0ab84d7032ef868076a6b813b79dd5f2292710f327e0bcb",
+ "error": null
+ },
+ "apply": null
+ },
+ "failure": {
+ "kind": "patch-invalid",
+ "name": "PatchError",
+ "message": "git apply --check rejected the patch",
+ "details": {
+ "checkRun": {
+ "argv": [
+ "git",
+ "apply",
+ "--check",
+ "--whitespace=error-all",
+ "/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-patch-83564-1785575176925-f9951cf8b0d45.diff"
+ ],
+ "cwd": "/private/var/folders/r0/2xc529t10dz2lk4749f0rh540000gn/T/sg-agent-task-mzh4Hq/workspace",
+ "status": 128,
+ "exitCode": 128,
+ "exit": 128,
+ "signal": null,
+ "timedOut": false,
+ "durationMs": 26.0165,
+ "duration": 26.0165,
+ "stdout": "",
+ "stderr": "error: patch fragment without header at line 20: @@ -22,5 +27,6 @@\n",
+ "stdoutTruncated": false,
+ "stderrTruncated": false,
+ "truncated": false
+ }
+ }
+ },
+ "capabilities": {
+ "osSandbox": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "sourceTreeVerifiedByRunner": true,
+ "disposableWorkspace": true
+ }
+ },
+ "grades": [],
+ "secondaryErrors": [],
+ "artifacts": [
+ {
+ "id": "agent-patch",
+ "path": "agent.patch",
+ "mediaType": "text/x-diff",
+ "bytes": 2537,
+ "sha256": "sha256:2649706bf415a7b28cf09ce7075f9d6d3017a869cf66e8015c13dda437d174e8"
+ },
+ {
+ "id": "patch-audit",
+ "path": "patch-audit.json",
+ "mediaType": "application/json",
+ "bytes": 2944,
+ "sha256": "sha256:35032c8e0365fd89860318ae676e064da3375c6494dd87f9f8c8e6183e940cb6"
+ },
+ {
+ "id": "task-prompt",
+ "path": "TASK.md",
+ "mediaType": "text/markdown",
+ "bytes": 368,
+ "sha256": "sha256:8d6d410df008eb20405838885369f435267b07b38267305d013cd4661c971d29"
+ }
+ ],
+ "verdict": "patch-invalid",
+ "termination": {
+ "kind": "completed",
+ "processExitCode": 1,
+ "semanticVerdict": "patch-invalid",
+ "message": "git apply --check rejected the patch"
+ },
+ "capabilities": {
+ "sourceCopy": true,
+ "disposableWorkspace": true,
+ "completeSecuritySnapshots": true,
+ "postAgentSourceTreeCheck": true,
+ "postPatchTreePolicy": true,
+ "postGraderTreeCheck": true,
+ "verifiedGraderStaging": true,
+ "verifiedToolStaging": true,
+ "candidateExtractionWorker": true,
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/execution-bundle.json b/research/agent-eval/results/executed-tooling/execution-bundle.json
new file mode 100644
index 0000000..02792e8
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/execution-bundle.json
@@ -0,0 +1,17 @@
+{
+ "bundleVersion": "1.0",
+ "kind": "executed-tooling",
+ "scriptsTreeSha256": "sha256:76864eb0e641bbe39bbd2694b0102985c23382150307e5710c558b26d2138283",
+ "source": "scripts/",
+ "usedBy": [
+ "scripted/experiment.raw.json",
+ "claude/experiment.raw.json"
+ ],
+ "capabilities": {
+ "osSandbox": false,
+ "filesystemIsolation": false,
+ "networkIsolation": false,
+ "processIsolation": false,
+ "maliciousAgentGraderSecrecy": false
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/agent-task-manifest.js b/research/agent-eval/results/executed-tooling/scripts/lib/agent-task-manifest.js
new file mode 100644
index 0000000..3aa428f
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/agent-task-manifest.js
@@ -0,0 +1,608 @@
+'use strict';
+
+/**
+ * Read-only AgentTaskManifest v1 contract helpers.
+ *
+ * This module intentionally does not depend on the path-policy package. It is a
+ * small, self-contained boundary for validating an already materialized task
+ * description and (optionally) checking the files it names.
+ */
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const TASK_VERSION = '1.0';
+const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/;
+const TOP_LEVEL_FIELDS = [
+ 'taskVersion', 'taskId', 'instructions', 'source', 'inputs',
+ 'filePolicy', 'patchPolicy', 'graders', 'evidence', 'execution',
+];
+const INSTRUCTION_FIELDS = ['text', 'sha256'];
+const SOURCE_FIELDS = ['root', 'revision', 'treeSha256'];
+const INPUT_FIELDS = ['id', 'path', 'role', 'sha256'];
+const FILE_POLICY_FIELDS = [
+ 'allowed', 'forbidden', 'denyPrecedence', 'allowSymlinks',
+ 'allowHardlinks', 'maxChangedFiles', 'maxPatchBytes',
+];
+const ALLOWED_FIELDS = ['pattern', 'operations'];
+const PATCH_POLICY_FIELDS = ['format', 'fuzz', 'allowBinary'];
+const GRADER_FIELDS = ['id', 'spec', 'weight', 'sha256', 'treeSha256'];
+const EVIDENCE_FIELDS = ['runtime', 'visual'];
+const EXECUTION_FIELDS = ['timeoutMs', 'network', 'maxOutputBytes'];
+
+// Bounds are deliberately finite so an otherwise valid JSON number cannot
+// disable a runner's resource gate through an unbounded integer.
+const RESOURCE_LIMITS = Object.freeze({
+ timeoutMs: 24 * 60 * 60 * 1000,
+ maxOutputBytes: 1024 * 1024 * 1024,
+ maxChangedFiles: 1000000,
+ maxPatchBytes: 1024 * 1024 * 1024,
+});
+
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function hasOwn(value, key) {
+ return Object.prototype.hasOwnProperty.call(value, key);
+}
+
+function ownKeys(value) {
+ return isObject(value) ? Object.keys(value) : [];
+}
+
+function issue(issues, pathName, message) {
+ issues.push({ path: pathName, message });
+}
+
+function checkExactKeys(value, allowed, pathName, issues) {
+ if (!isObject(value)) return;
+ const allowedSet = new Set(allowed);
+ for (const key of Object.keys(value)) {
+ if (!allowedSet.has(key)) issue(issues, `${pathName}.${key}`, 'unknown field');
+ }
+}
+
+function requireObject(value, pathName, issues) {
+ if (!isObject(value)) {
+ issue(issues, pathName, 'must be an object');
+ return false;
+ }
+ return true;
+}
+
+function requireArray(value, pathName, issues) {
+ if (!Array.isArray(value)) {
+ issue(issues, pathName, 'must be an array');
+ return false;
+ }
+ return true;
+}
+
+function requireString(value, pathName, issues, options = {}) {
+ if (typeof value !== 'string' || (options.nonEmpty !== false && value.length === 0)) {
+ issue(issues, pathName, options.message || (options.nonEmpty === false ? 'must be a string' : 'must be a non-empty string'));
+ return false;
+ }
+ if (value.includes('\0')) issue(issues, pathName, 'must not contain NUL');
+ return true;
+}
+
+function requireDigest(value, pathName, issues) {
+ if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) {
+ issue(issues, pathName, 'must be a sha256:<64 lowercase hex> digest');
+ return false;
+ }
+ return true;
+}
+
+/**
+ * Canonical JSON sorts object keys recursively and preserves array order.
+ * JSON.stringify supplies the stable compact UTF-8 material that is hashed.
+ */
+function canonicalize(value) {
+ if (Array.isArray(value)) return value.map(canonicalize);
+ if (isObject(value)) {
+ return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
+ }
+ return value;
+}
+
+function canonicalJson(value) {
+ return JSON.stringify(canonicalize(value));
+}
+
+function sha256(value) {
+ const bytes = Buffer.isBuffer(value) ? value : Buffer.from(String(value), 'utf8');
+ return `sha256:${crypto.createHash('sha256').update(bytes).digest('hex')}`;
+}
+
+function withoutTaskId(manifest) {
+ if (!isObject(manifest)) return manifest;
+ const material = { ...manifest };
+ delete material.taskId;
+ return material;
+}
+
+function computeTaskId(manifest) {
+ if (!isObject(manifest)) throw new TypeError('manifest must be an object');
+ return sha256(canonicalJson(withoutTaskId(manifest)));
+}
+
+function validateSafeRelativePath(value, pathName, issues, options = {}) {
+ if (!requireString(value, pathName, issues)) return false;
+ if (value.includes('\\')) issue(issues, pathName, 'must use POSIX separators and must not contain backslash');
+ if (options.allowGlob !== true && /[*?{}\[\]]/.test(value)) issue(issues, pathName, 'must be a concrete path, not a glob pattern');
+ if (path.posix.isAbsolute(value) || /^\/+/.test(value) || /^[A-Za-z]:/.test(value)) {
+ issue(issues, pathName, 'must be a relative path');
+ }
+ const segments = value.split('/');
+ if (segments.some((segment) => segment === '..')) issue(issues, pathName, 'must not contain .. path segments');
+ if (segments.some((segment) => segment === '')) issue(issues, pathName, 'must not contain empty path segments');
+ const dotSegments = segments.filter((segment) => segment === '.');
+ if (dotSegments.length && !(options.allowDot === 'self' && value === '.')) {
+ issue(issues, pathName, 'must not contain . path segments');
+ }
+ return true;
+}
+
+function checkDuplicateStrings(values, pathName, issues) {
+ const seen = new Map();
+ values.forEach((value, index) => {
+ if (typeof value !== 'string') return;
+ if (seen.has(value)) issue(issues, `${pathName}[${index}]`, `duplicate value; first occurrence is ${pathName}[${seen.get(value)}]`);
+ else seen.set(value, index);
+ });
+}
+
+function checkDuplicateIds(items, pathName, issues) {
+ const seen = new Map();
+ items.forEach((item, index) => {
+ if (!isObject(item) || typeof item.id !== 'string') return;
+ if (seen.has(item.id)) issue(issues, `${pathName}[${index}].id`, `duplicate id; first occurrence is ${pathName}[${seen.get(item.id)}].id`);
+ else seen.set(item.id, index);
+ });
+}
+
+function validateInstructions(value, issues) {
+ if (!requireObject(value, 'instructions', issues)) return;
+ checkExactKeys(value, INSTRUCTION_FIELDS, 'instructions', issues);
+ requireString(value.text, 'instructions.text', issues);
+ requireDigest(value.sha256, 'instructions.sha256', issues);
+ if (typeof value.text === 'string' && typeof value.sha256 === 'string' && DIGEST_PATTERN.test(value.sha256)) {
+ const expected = sha256(Buffer.from(value.text, 'utf8'));
+ if (expected !== value.sha256) issue(issues, 'instructions.sha256', `digest does not match instructions.text (expected ${expected})`);
+ }
+}
+
+function validateSource(value, issues) {
+ if (!requireObject(value, 'source', issues)) return;
+ checkExactKeys(value, SOURCE_FIELDS, 'source', issues);
+ validateSafeRelativePath(value.root, 'source.root', issues, { allowDot: 'self' });
+ requireString(value.revision, 'source.revision', issues);
+ requireDigest(value.treeSha256, 'source.treeSha256', issues);
+}
+
+function validateInputs(value, issues) {
+ if (!requireArray(value, 'inputs', issues)) return;
+ checkDuplicateIds(value, 'inputs', issues);
+ value.forEach((input, index) => {
+ const base = `inputs[${index}]`;
+ if (!requireObject(input, base, issues)) return;
+ checkExactKeys(input, INPUT_FIELDS, base, issues);
+ requireString(input.id, `${base}.id`, issues);
+ validateSafeRelativePath(input.path, `${base}.path`, issues);
+ requireString(input.role, `${base}.role`, issues);
+ requireDigest(input.sha256, `${base}.sha256`, issues);
+ });
+}
+
+function validateFilePolicy(value, issues) {
+ if (!requireObject(value, 'filePolicy', issues)) return;
+ checkExactKeys(value, FILE_POLICY_FIELDS, 'filePolicy', issues);
+ if (!requireArray(value.allowed, 'filePolicy.allowed', issues)) return;
+ if (!requireArray(value.forbidden, 'filePolicy.forbidden', issues)) return;
+ if (value.denyPrecedence !== true) issue(issues, 'filePolicy.denyPrecedence', 'must be true');
+ if (value.allowSymlinks !== false) issue(issues, 'filePolicy.allowSymlinks', 'must be false');
+ if (value.allowHardlinks !== false) issue(issues, 'filePolicy.allowHardlinks', 'must be false');
+ validateResourceInteger(value.maxChangedFiles, 'filePolicy.maxChangedFiles', RESOURCE_LIMITS.maxChangedFiles, issues);
+ validateResourceInteger(value.maxPatchBytes, 'filePolicy.maxPatchBytes', RESOURCE_LIMITS.maxPatchBytes, issues);
+
+ const patterns = [];
+ value.allowed.forEach((rule, index) => {
+ const base = `filePolicy.allowed[${index}]`;
+ if (!requireObject(rule, base, issues)) return;
+ checkExactKeys(rule, ALLOWED_FIELDS, base, issues);
+ validateSafeRelativePath(rule.pattern, `${base}.pattern`, issues, { allowDot: true, allowGlob: true });
+ if (!requireArray(rule.operations, `${base}.operations`, issues)) return;
+ if (rule.operations.length === 0) issue(issues, `${base}.operations`, 'must contain at least one operation');
+ checkDuplicateStrings(rule.operations, `${base}.operations`, issues);
+ rule.operations.forEach((operation, operationIndex) => {
+ if (!['add', 'modify', 'delete'].includes(operation)) issue(issues, `${base}.operations[${operationIndex}]`, 'must be add, modify, or delete');
+ });
+ if (typeof rule.pattern === 'string') patterns.push({ pattern: rule.pattern, index });
+ });
+ const seenPatterns = new Map();
+ patterns.forEach(({ pattern, index }) => {
+ if (seenPatterns.has(pattern)) issue(issues, `filePolicy.allowed[${index}].pattern`, `duplicate pattern; first occurrence is ${seenPatterns.get(pattern).path}`);
+ else seenPatterns.set(pattern, { path: `filePolicy.allowed[${index}].pattern`, index });
+ });
+ checkDuplicateStrings(value.forbidden, 'filePolicy.forbidden', issues);
+ value.forbidden.forEach((pattern, index) => {
+ validateSafeRelativePath(pattern, `filePolicy.forbidden[${index}]`, issues, { allowDot: true, allowGlob: true });
+ if (typeof pattern === 'string' && seenPatterns.has(pattern)) {
+ issue(issues, `filePolicy.forbidden[${index}]`, `duplicate pattern; first occurrence is ${seenPatterns.get(pattern).path}`);
+ } else if (typeof pattern === 'string') {
+ seenPatterns.set(pattern, { path: `filePolicy.forbidden[${index}]`, index });
+ }
+ });
+}
+
+function validatePatchPolicy(value, issues) {
+ if (!requireObject(value, 'patchPolicy', issues)) return;
+ checkExactKeys(value, PATCH_POLICY_FIELDS, 'patchPolicy', issues);
+ if (value.format !== 'unified-diff') issue(issues, 'patchPolicy.format', 'must be unified-diff');
+ if (value.fuzz !== 0) issue(issues, 'patchPolicy.fuzz', 'must be 0');
+ if (value.allowBinary !== false) issue(issues, 'patchPolicy.allowBinary', 'must be false');
+}
+
+function validateGraders(value, issues) {
+ if (!requireArray(value, 'graders', issues)) return;
+ checkDuplicateIds(value, 'graders', issues);
+ value.forEach((grader, index) => {
+ const base = `graders[${index}]`;
+ if (!requireObject(grader, base, issues)) return;
+ checkExactKeys(grader, GRADER_FIELDS, base, issues);
+ requireString(grader.id, `${base}.id`, issues);
+ if (!hasOwn(grader, 'spec') || grader.spec === undefined || grader.spec === null) issue(issues, `${base}.spec`, 'is required and must not be null');
+ if (typeof grader.spec === 'string') validateSafeRelativePath(grader.spec, `${base}.spec`, issues);
+ else issue(issues, `${base}.spec`, 'must be a relative path to a grader spec');
+ requireDigest(grader.sha256, `${base}.sha256`, issues);
+ requireDigest(grader.treeSha256, `${base}.treeSha256`, issues);
+ if (typeof grader.weight !== 'number' || !Number.isFinite(grader.weight) || grader.weight <= 0) issue(issues, `${base}.weight`, 'must be a finite number greater than 0');
+ });
+}
+
+function validateEvidence(value, issues) {
+ if (!requireObject(value, 'evidence', issues)) return;
+ checkExactKeys(value, EVIDENCE_FIELDS, 'evidence', issues);
+ for (const kind of EVIDENCE_FIELDS) {
+ if (!requireArray(value[kind], `evidence.${kind}`, issues)) continue;
+ checkDuplicateStrings(value[kind], `evidence.${kind}`, issues);
+ value[kind].forEach((scenarioId, index) => requireString(scenarioId, `evidence.${kind}[${index}]`, issues));
+ }
+}
+
+function validateResourceInteger(value, pathName, maximum, issues) {
+ if (!Number.isSafeInteger(value) || value < 0 || value > maximum) {
+ issue(issues, pathName, `must be an integer from 0 to ${maximum}`);
+ }
+}
+
+function validateExecution(value, issues) {
+ if (!requireObject(value, 'execution', issues)) return;
+ checkExactKeys(value, EXECUTION_FIELDS, 'execution', issues);
+ if (!Number.isSafeInteger(value.timeoutMs) || value.timeoutMs < 1 || value.timeoutMs > RESOURCE_LIMITS.timeoutMs) {
+ issue(issues, 'execution.timeoutMs', `must be an integer from 1 to ${RESOURCE_LIMITS.timeoutMs}`);
+ }
+ if (value.network !== 'off') issue(issues, 'execution.network', 'must be off');
+ if (!Number.isSafeInteger(value.maxOutputBytes) || value.maxOutputBytes < 1 || value.maxOutputBytes > RESOURCE_LIMITS.maxOutputBytes) {
+ issue(issues, 'execution.maxOutputBytes', `must be an integer from 1 to ${RESOURCE_LIMITS.maxOutputBytes}`);
+ }
+}
+
+function resolveTaskFile(taskFile) {
+ if (typeof taskFile !== 'string' || taskFile.length === 0) return null;
+ return path.resolve(taskFile);
+}
+
+function resolveSourceRoot(manifest, options = {}) {
+ if (!isObject(manifest) || !isObject(manifest.source) || typeof manifest.source.root !== 'string') return null;
+ const taskFile = resolveTaskFile(options.taskFile);
+ if (taskFile) return path.resolve(path.dirname(taskFile), manifest.source.root);
+ if (typeof options.sourceRoot === 'string' && options.sourceRoot.length > 0) return path.resolve(options.sourceRoot);
+ return null;
+}
+
+function isWithinRoot(root, target) {
+ const relative = path.relative(root, target);
+ return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
+}
+
+function lstatPathParts(root, relativePath) {
+ const parts = relativePath.split('/');
+ const result = [];
+ let current = root;
+ for (const part of parts) {
+ current = path.join(current, part);
+ result.push({ path: current, stat: fs.lstatSync(current) });
+ }
+ return result;
+}
+
+function verifyGraderFiles(manifest, taskFile, issues) {
+ if (!taskFile) {
+ issue(issues, '$options.taskFile', 'is required to verify grader files');
+ return;
+ }
+ const taskDir = path.dirname(taskFile);
+ const requiredEvidence = { runtime: new Set(), visual: new Set() };
+ for (const [index, grader] of manifest.graders.entries()) {
+ if (!isObject(grader) || typeof grader.spec !== 'string') continue;
+ const specFile = path.resolve(taskDir, ...grader.spec.split('/'));
+ if (!isWithinRoot(taskDir, specFile)) {
+ issue(issues, `graders[${index}].spec`, 'resolves outside task directory');
+ continue;
+ }
+ try {
+ const stat = fs.lstatSync(specFile);
+ if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('must be a regular non-symlink file');
+ if (stat.nlink > 1) throw new Error('hardlinks are not allowed');
+ const bytes = fs.readFileSync(specFile);
+ const actual = sha256(bytes);
+ if (actual !== grader.sha256) issue(issues, `graders[${index}].sha256`, `digest is stale (actual ${actual})`);
+ const actualTree = computeTreeSha256(path.dirname(specFile));
+ if (actualTree !== grader.treeSha256) issue(issues, `graders[${index}].treeSha256`, `digest is stale (actual ${actualTree})`);
+ const spec = JSON.parse(bytes.toString('utf8'));
+ for (const check of Array.isArray(spec.checks) ? spec.checks : []) {
+ const match = /^(runtime|visual)-evidence$/.exec(check && check.type);
+ if (match && check.required === true && typeof check.scenarioId === 'string' && check.scenarioId) requiredEvidence[match[1]].add(check.scenarioId);
+ }
+ } catch (error) {
+ issue(issues, `graders[${index}].spec`, `cannot verify grader: ${error.message}`);
+ }
+ }
+ for (const kind of ['runtime', 'visual']) {
+ const declared = new Set(manifest.evidence && Array.isArray(manifest.evidence[kind]) ? manifest.evidence[kind] : []);
+ for (const scenarioId of declared) if (!requiredEvidence[kind].has(scenarioId)) issue(issues, `evidence.${kind}`, `declared scenario has no required grader check: ${scenarioId}`);
+ for (const scenarioId of requiredEvidence[kind]) if (!declared.has(scenarioId)) issue(issues, `evidence.${kind}`, `required grader scenario is not declared: ${scenarioId}`);
+ }
+}
+
+function verifyInputFiles(manifest, sourceRoot, issues) {
+ if (!sourceRoot) {
+ issue(issues, '$options.sourceRoot', 'is required when verifyFiles is true');
+ return;
+ }
+ if (!Array.isArray(manifest.inputs)) return;
+ if (!isObject(manifest.filePolicy)) return;
+ let rootStat;
+ try {
+ rootStat = fs.lstatSync(sourceRoot);
+ } catch (error) {
+ issue(issues, 'source.root', `cannot read source root: ${error.message}`);
+ return;
+ }
+ if (rootStat.isSymbolicLink() && manifest.filePolicy.allowSymlinks === false) {
+ issue(issues, 'source.root', 'symlinks are not allowed');
+ return;
+ }
+ if (!rootStat.isDirectory()) {
+ issue(issues, 'source.root', 'must resolve to a directory');
+ return;
+ }
+ for (const [index, input] of manifest.inputs.entries()) {
+ if (!isObject(input) || typeof input.path !== 'string') continue;
+ const inputPath = path.resolve(sourceRoot, ...input.path.split('/'));
+ if (!isWithinRoot(sourceRoot, inputPath)) {
+ issue(issues, `inputs[${index}].path`, 'resolves outside source root');
+ continue;
+ }
+ let parts;
+ try {
+ parts = lstatPathParts(sourceRoot, input.path);
+ } catch (error) {
+ issue(issues, `inputs[${index}].path`, `cannot read input: ${error.message}`);
+ continue;
+ }
+ const symlink = parts.find((part) => part.stat.isSymbolicLink());
+ if (symlink && manifest.filePolicy.allowSymlinks === false) {
+ issue(issues, `inputs[${index}].path`, 'symlinks are not allowed');
+ continue;
+ }
+ const stat = parts[parts.length - 1].stat;
+ if (!stat.isFile()) {
+ issue(issues, `inputs[${index}].path`, 'must resolve to a regular file');
+ continue;
+ }
+ if (manifest.filePolicy.allowHardlinks === false && stat.nlink > 1) {
+ issue(issues, `inputs[${index}].path`, 'hardlinks are not allowed');
+ continue;
+ }
+ let bytes;
+ try {
+ bytes = fs.readFileSync(inputPath);
+ } catch (error) {
+ issue(issues, `inputs[${index}].path`, `cannot read input: ${error.message}`);
+ continue;
+ }
+ const actual = sha256(bytes);
+ if (actual !== input.sha256) issue(issues, `inputs[${index}].sha256`, `digest is stale (actual ${actual})`);
+ }
+}
+
+function collectTreeEntries(root, relative, manifest, entries, issues) {
+ let directoryEntries;
+ try {
+ directoryEntries = fs.readdirSync(path.join(root, relative), { withFileTypes: true });
+ } catch (error) {
+ issue(issues, 'source.treeSha256', `cannot read source tree: ${error.message}`);
+ return;
+ }
+ directoryEntries.sort((a, b) => Buffer.from(a.name).compare(Buffer.from(b.name)));
+ for (const directoryEntry of directoryEntries) {
+ const childRelative = relative ? `${relative}/${directoryEntry.name}` : directoryEntry.name;
+ const childPath = path.join(root, childRelative);
+ let stat;
+ try {
+ stat = fs.lstatSync(childPath);
+ } catch (error) {
+ issue(issues, 'source.treeSha256', `cannot stat ${childRelative}: ${error.message}`);
+ continue;
+ }
+ if (stat.isSymbolicLink()) {
+ if (manifest.filePolicy.allowSymlinks === false) issue(issues, `source.treeSha256`, `symlinks are not allowed: ${childRelative}`);
+ else {
+ let target;
+ try { target = fs.readlinkSync(childPath); } catch (error) { target = ``; }
+ entries.push({ path: childRelative, type: 'symlink', mode: stat.mode & 0o7777, target });
+ }
+ continue;
+ }
+ if (stat.isDirectory()) {
+ entries.push({ path: childRelative, type: 'directory', mode: stat.mode & 0o7777 });
+ collectTreeEntries(root, childRelative, manifest, entries, issues);
+ continue;
+ }
+ if (stat.isFile()) {
+ if (manifest.filePolicy.allowHardlinks === false && stat.nlink > 1) issue(issues, 'source.treeSha256', `hardlinks are not allowed: ${childRelative}`);
+ try {
+ entries.push({ path: childRelative, type: 'file', mode: stat.mode & 0o7777, sha256: sha256(fs.readFileSync(childPath)) });
+ } catch (error) {
+ issue(issues, 'source.treeSha256', `cannot read ${childRelative}: ${error.message}`);
+ }
+ continue;
+ }
+ issue(issues, 'source.treeSha256', `unsupported filesystem entry: ${childRelative}`);
+ }
+}
+
+function computeTreeSha256(sourceRoot, options = {}) {
+ const root = path.resolve(sourceRoot);
+ let rootStat;
+ try {
+ rootStat = fs.lstatSync(root);
+ } catch (error) {
+ const readError = new Error(`cannot read source tree root: ${error.message}`);
+ readError.code = 'TREE_DIGEST_ERROR';
+ throw readError;
+ }
+ if (rootStat.isSymbolicLink() && options.allowSymlinks !== true) {
+ const linkError = new Error(`source tree root is a symlink: ${root}`);
+ linkError.code = 'TREE_DIGEST_ERROR';
+ throw linkError;
+ }
+ if (!rootStat.isDirectory()) {
+ const directoryError = new Error(`source tree root must be a directory: ${root}`);
+ directoryError.code = 'TREE_DIGEST_ERROR';
+ throw directoryError;
+ }
+ const entries = [];
+ const manifest = { filePolicy: {
+ allowSymlinks: options.allowSymlinks === true,
+ allowHardlinks: options.allowHardlinks === true,
+ } };
+ const issues = [];
+ collectTreeEntries(root, '', manifest, entries, issues);
+ if (issues.length) {
+ const error = new Error(`cannot compute source tree digest: ${issues.map((item) => item.message).join('; ')}`);
+ error.code = 'TREE_DIGEST_ERROR';
+ error.issues = issues;
+ throw error;
+ }
+ return sha256(canonicalJson(entries));
+}
+
+function validateTaskManifest(manifest, options = {}) {
+ const issues = [];
+ if (!isObject(manifest)) return { valid: false, issues: [{ path: '$', message: 'manifest must be an object' }] };
+ checkExactKeys(manifest, TOP_LEVEL_FIELDS, '$', issues);
+ for (const field of TOP_LEVEL_FIELDS) {
+ if (!hasOwn(manifest, field)) issue(issues, field, 'is required');
+ }
+ if (manifest.taskVersion !== TASK_VERSION) issue(issues, 'taskVersion', 'must be 1.0');
+ requireDigest(manifest.taskId, 'taskId', issues);
+ validateInstructions(manifest.instructions, issues);
+ validateSource(manifest.source, issues);
+ validateInputs(manifest.inputs, issues);
+ validateFilePolicy(manifest.filePolicy, issues);
+ validatePatchPolicy(manifest.patchPolicy, issues);
+ validateGraders(manifest.graders, issues);
+ validateEvidence(manifest.evidence, issues);
+ validateExecution(manifest.execution, issues);
+
+ let expectedTaskId;
+ try {
+ expectedTaskId = computeTaskId(manifest);
+ if (typeof manifest.taskId === 'string' && DIGEST_PATTERN.test(manifest.taskId) && manifest.taskId !== expectedTaskId) {
+ issue(issues, 'taskId', `does not match canonical manifest material (expected ${expectedTaskId})`);
+ }
+ } catch (error) {
+ issue(issues, 'taskId', `cannot compute taskId: ${error.message}`);
+ }
+
+ const taskFile = resolveTaskFile(options.taskFile);
+ const resolvedSourceRoot = resolveSourceRoot(manifest, options);
+ if (options.sourceRoot !== undefined) {
+ if (typeof options.sourceRoot !== 'string' || options.sourceRoot.length === 0) issue(issues, '$options.sourceRoot', 'must be a non-empty path');
+ else if (resolvedSourceRoot && path.resolve(options.sourceRoot) !== resolvedSourceRoot) issue(issues, '$options.sourceRoot', `does not match source.root resolution (${resolvedSourceRoot})`);
+ }
+ if (options.verifyFiles === true) {
+ verifyInputFiles(manifest, resolvedSourceRoot, issues);
+ verifyGraderFiles(manifest, taskFile, issues);
+ }
+ if (options.verifyTree === true) {
+ if (!resolvedSourceRoot) issue(issues, '$options.sourceRoot', 'or taskFile is required when verifyTree is true');
+ else {
+ try {
+ const actual = computeTreeSha256(resolvedSourceRoot, {
+ allowSymlinks: manifest.filePolicy && manifest.filePolicy.allowSymlinks === true,
+ allowHardlinks: manifest.filePolicy && manifest.filePolicy.allowHardlinks === true,
+ });
+ if (actual !== manifest.source.treeSha256) issue(issues, 'source.treeSha256', `digest is stale (actual ${actual})`);
+ } catch (error) {
+ issue(issues, 'source.treeSha256', error.message);
+ }
+ }
+ }
+
+ return {
+ valid: issues.length === 0,
+ issues,
+ taskId: manifest.taskId,
+ expectedTaskId,
+ taskFile,
+ sourceRoot: resolvedSourceRoot,
+ };
+}
+
+function invalidManifestError(result, file) {
+ const error = new Error(`invalid AgentTaskManifest${file ? ` ${file}` : ''}: ${result.issues.map((item) => `${item.path}: ${item.message}`).join('; ')}`);
+ error.code = 'INVALID_AGENT_TASK_MANIFEST';
+ error.issues = result.issues;
+ error.validation = result;
+ return error;
+}
+
+function readTaskManifest(file, options = {}) {
+ if (typeof file !== 'string' || file.length === 0) throw new TypeError('manifest file path is required');
+ const taskFile = path.resolve(file);
+ let manifest;
+ try {
+ manifest = JSON.parse(fs.readFileSync(taskFile, 'utf8'));
+ } catch (error) {
+ const readError = new Error(`cannot read AgentTaskManifest ${taskFile}: ${error.message}`);
+ readError.code = 'AGENT_TASK_MANIFEST_READ_ERROR';
+ readError.cause = error;
+ throw readError;
+ }
+ const result = validateTaskManifest(manifest, { ...options, taskFile });
+ if (!result.valid) throw invalidManifestError(result, taskFile);
+ return manifest;
+}
+
+module.exports = {
+ TASK_VERSION,
+ DIGEST_PATTERN,
+ TOP_LEVEL_FIELDS,
+ RESOURCE_LIMITS,
+ canonicalize,
+ canonicalJson,
+ stableJson: canonicalJson,
+ sha256,
+ computeTaskId,
+ computeTreeSha256,
+ resolveSourceRoot,
+ validateTaskManifest,
+ readTaskManifest,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/agent-task.schema.json b/research/agent-eval/results/executed-tooling/scripts/lib/agent-task.schema.json
new file mode 100644
index 0000000..97558c0
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/agent-task.schema.json
@@ -0,0 +1,181 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://sg.local/agent-task.schema.json",
+ "title": "AgentTaskManifest v1",
+ "description": "Declarative, content-addressed task contract for an agent execution.",
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "taskVersion",
+ "taskId",
+ "instructions",
+ "source",
+ "inputs",
+ "filePolicy",
+ "patchPolicy",
+ "graders",
+ "evidence",
+ "execution"
+ ],
+ "properties": {
+ "taskVersion": { "const": "1.0" },
+ "taskId": { "$ref": "#/$defs/digest" },
+ "instructions": { "$ref": "#/$defs/instructions" },
+ "source": { "$ref": "#/$defs/source" },
+ "inputs": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/input" },
+ "uniqueItems": true
+ },
+ "filePolicy": { "$ref": "#/$defs/filePolicy" },
+ "patchPolicy": { "$ref": "#/$defs/patchPolicy" },
+ "graders": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/grader" },
+ "uniqueItems": true
+ },
+ "evidence": { "$ref": "#/$defs/evidence" },
+ "execution": { "$ref": "#/$defs/execution" }
+ },
+ "$defs": {
+ "digest": {
+ "type": "string",
+ "pattern": "^sha256:[a-f0-9]{64}$"
+ },
+ "relativePath": {
+ "type": "string",
+ "minLength": 1,
+ "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*\\u0000)(?!.*[*?{}\\[\\]])(?!.*(?:^|/)\\.\\.?(?:/|$))(?!.*//).+"
+ },
+ "relativePattern": {
+ "type": "string",
+ "minLength": 1,
+ "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*\\u0000)(?!.*(?:^|/)\\.\\.?(?:/|$))(?!.*//).+"
+ },
+ "relativeRoot": {
+ "type": "string",
+ "minLength": 1,
+ "pattern": "^(?:\\.|(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*\\u0000)(?!.*[*?{}\\[\\]])(?!.*(?:^|/)\\.\\.?(?:/|$))(?!.*//).+)$"
+ },
+ "instructions": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["text", "sha256"],
+ "properties": {
+ "text": { "type": "string", "minLength": 1 },
+ "sha256": { "$ref": "#/$defs/digest" }
+ }
+ },
+ "source": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["root", "revision", "treeSha256"],
+ "properties": {
+ "root": { "$ref": "#/$defs/relativeRoot" },
+ "revision": { "type": "string", "minLength": 1 },
+ "treeSha256": { "$ref": "#/$defs/digest" }
+ }
+ },
+ "input": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["id", "path", "role", "sha256"],
+ "properties": {
+ "id": { "type": "string", "minLength": 1 },
+ "path": { "$ref": "#/$defs/relativePath" },
+ "role": { "type": "string", "minLength": 1 },
+ "sha256": { "$ref": "#/$defs/digest" }
+ }
+ },
+ "filePolicy": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "allowed",
+ "forbidden",
+ "denyPrecedence",
+ "allowSymlinks",
+ "allowHardlinks",
+ "maxChangedFiles",
+ "maxPatchBytes"
+ ],
+ "properties": {
+ "allowed": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/allowedRule" },
+ "uniqueItems": true
+ },
+ "forbidden": {
+ "type": "array",
+ "items": { "$ref": "#/$defs/relativePattern" },
+ "uniqueItems": true
+ },
+ "denyPrecedence": { "const": true },
+ "allowSymlinks": { "const": false },
+ "allowHardlinks": { "const": false },
+ "maxChangedFiles": { "$ref": "#/$defs/nonNegativeBoundedInteger" },
+ "maxPatchBytes": { "$ref": "#/$defs/nonNegativeBoundedInteger" }
+ }
+ },
+ "allowedRule": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["pattern", "operations"],
+ "properties": {
+ "pattern": { "$ref": "#/$defs/relativePattern" },
+ "operations": {
+ "type": "array",
+ "minItems": 1,
+ "items": { "enum": ["add", "modify", "delete"] },
+ "uniqueItems": true
+ }
+ }
+ },
+ "patchPolicy": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["format", "fuzz", "allowBinary"],
+ "properties": {
+ "format": { "const": "unified-diff" },
+ "fuzz": { "const": 0 },
+ "allowBinary": { "const": false }
+ }
+ },
+ "grader": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["id", "spec", "sha256", "treeSha256", "weight"],
+ "properties": {
+ "id": { "type": "string", "minLength": 1 },
+ "spec": { "$ref": "#/$defs/relativePath" },
+ "sha256": { "$ref": "#/$defs/digest" },
+ "treeSha256": { "$ref": "#/$defs/digest" },
+ "weight": { "type": "number", "exclusiveMinimum": 0, "maximum": 1000000 }
+ }
+ },
+ "evidence": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["runtime", "visual"],
+ "properties": {
+ "runtime": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } },
+ "visual": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } }
+ }
+ },
+ "execution": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["timeoutMs", "network", "maxOutputBytes"],
+ "properties": {
+ "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 86400000 },
+ "network": { "const": "off" },
+ "maxOutputBytes": { "type": "integer", "minimum": 1, "maximum": 1073741824 }
+ }
+ },
+ "nonNegativeBoundedInteger": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 1073741824
+ }
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/data-pack.schema.json b/research/agent-eval/results/executed-tooling/scripts/lib/data-pack.schema.json
new file mode 100644
index 0000000..4e78c60
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/data-pack.schema.json
@@ -0,0 +1,554 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://sg.local/data-pack.schema.json",
+ "title": "SG Data Pack v1.3 — Unified Data Contract for Component Libraries",
+ "description": "The single entry point for business data of all sg-* component libraries. Globally unique entities, id references, stages that reference rather than duplicate. Runtime validation is performed by sg-data-loader.js (rules E1-E16/W1-W8 correspond one-to-one with the rules below); this file is the formal contract. v1.1 added: relations.scope, attributeTypes/attributeSources, contents, assets hash, meta.assetBase, kindNameFields. v1.2 added: sameAs (entity identity), provenance (record-level provenance as a parallel section that never invades entity bodies — zero engine changes), meta.confidenceThreshold, W6 fuzzy duplicate detection. v1.3 added: derivations contract (kind/source/consumers/note/alsoTouches/affects, E16-validated), E11 domain asset coverage, contextual alias {id,context} runtime resolution, E12 scope canonicalization, provenance W7/W8 warnings.",
+ "type": "object",
+ "required": [
+ "schemaVersion",
+ "meta",
+ "entities"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "schemaVersion": {
+ "enum": [
+ "1.0",
+ "1.1",
+ "1.2",
+ "1.3"
+ ]
+ },
+ "meta": {
+ "type": "object",
+ "required": [
+ "id",
+ "title"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Library identifier, matching the directory name"
+ },
+ "title": {
+ "type": "string",
+ "minLength": 1
+ },
+ "hero": {
+ "type": "string",
+ "description": "Core entity id (graph center / protagonist); must exist in entities"
+ },
+ "source": {
+ "type": "string",
+ "description": "Data source (crawl-source annotation)"
+ },
+ "fetchedAt": {
+ "type": "string",
+ "description": "Fetch date, YYYY-MM-DD"
+ },
+ "generatedBy": {
+ "type": "string",
+ "description": "Extraction-script path (@generated marker)"
+ },
+ "assetBase": {
+ "type": "string",
+ "description": "Resolution prefix for bare-filename assets (e.g. '../assets/'); once declared, 'x.png' in the data resolves and registers as assetBase+'x.png'"
+ },
+ "confidenceThreshold": {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1,
+ "description": "(v1.2) Low-confidence warning threshold; default 0.7"
+ },
+ "recrawlFieldMap": {
+ "type": "object",
+ "description": "(v1.3) Optional crawled-field -> entity-field mapping used by recrawl-skeleton for like-for-like cross-checks (e.g. {\"birthDate\":\"birth\"}). Without it, recrawl does not guess field correspondences.",
+ "additionalProperties": {
+ "type": "string",
+ "minLength": 1
+ }
+ }
+ },
+ "additionalProperties": true
+ },
+ "kindNameFields": {
+ "type": "object",
+ "description": "kind -> name-field mapping (e.g. {work: 'title'}); kinds not declared here default to the name field",
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "sameAs": {
+ "type": "array",
+ "description": "(v1.2) Entity-identity declarations: [[idA, idB], ...] means two entities refer to the same real-world subject (e.g. Ying Zheng ↔ Qin Shi Huang). Once declared, W6 no longer flags the pair; downstream dedup/aggregation relies on this.",
+ "items": {
+ "type": "array",
+ "prefixItems": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "string"
+ }
+ ],
+ "minItems": 2,
+ "maxItems": 2
+ }
+ },
+ "provenance": {
+ "type": "object",
+ "description": "(v1.2) Record-level provenance (parallel-section design: never embedded into entities/relations/contents bodies, so engines need no changes). Each group's keys point at the corresponding records; confidence below meta.confidenceThreshold triggers a W5 low-confidence warning.",
+ "properties": {
+ "entities": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/$defs/provEntry"
+ }
+ },
+ "relations": {
+ "type": "object",
+ "description": "Key format: 'a::b' (entity-id pair)",
+ "additionalProperties": {
+ "$ref": "#/$defs/provEntry"
+ }
+ },
+ "contents": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/$defs/provEntry"
+ }
+ }
+ },
+ "additionalProperties": false
+ },
+ "entities": {
+ "type": "object",
+ "minProperties": 1,
+ "description": "Global entity table. Keys are safe stable ids (semantic slugs); values preserve all business fields of the entity. Referencing by name or array index is forbidden. The formal schema requires kind; runtime E3 additionally requires the display field selected by kindNameFields[kind], defaulting to name, because that cross-field lookup is not expressed by this generic schema.",
+ "propertyNames": {
+ "pattern": "^(?!__proto__$|prototype$|constructor$)[a-z][a-z0-9_-]*$"
+ },
+ "patternProperties": {
+ "^(?!__proto__$|prototype$|constructor$)[a-z][a-z0-9_-]*$": {
+ "type": "object",
+ "required": [
+ "kind"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Default display field (may be Chinese; for display only, never a reference key). Runtime E3 requires this unless kindNameFields maps the entity kind to another non-empty string field."
+ },
+ "kind": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Entity kind: person / work / term / event / ..."
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "additionalProperties": false
+ },
+ "aliases": {
+ "type": "object",
+ "description": "Crawled name / variant spelling -> canonical id. The normalization entry point on the data-production side; resolvable at runtime via SGDataLoader.resolveId. Values may be a plain id string or a {id, context} object for same-name disambiguation (E9 reserved context).",
+ "propertyNames": {
+ "not": { "enum": ["__proto__", "prototype", "constructor"] }
+ },
+ "additionalProperties": {
+ "oneOf": [
+ { "type": "string", "minLength": 1, "description": "Canonical entity id" },
+ {
+ "type": "object",
+ "required": ["id"],
+ "properties": {
+ "id": { "type": "string", "minLength": 1, "description": "Canonical entity id" },
+ "context": { "type": "string", "description": "Disambiguation context (e.g. 'cast', 'crew')" }
+ },
+ "additionalProperties": true
+ }
+ ]
+ }
+ },
+ "relationTypes": {
+ "type": "object",
+ "description": "Relationship-type registry (enum + display attributes). Every relations[].type must be registered here.",
+ "additionalProperties": {
+ "type": "object",
+ "required": [
+ "label"
+ ],
+ "properties": {
+ "label": {
+ "type": "string",
+ "minLength": 1
+ },
+ "color": {
+ "type": "string",
+ "description": "Color value or CSS variable reference"
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "heroRelTypes": {
+ "type": "object",
+ "description": "(optional, Plan-B dual registry) Relationship-type registry for the hero-radial dimension (person -> core). When overlay.rel / heroRel and relations[].type are semantically different dimensions, register them separately here and in relationTypes, with library-level rules forbidding cross-dimension use (e.g. qinshihuang: overlay.rel in {ruler, family, enemy})",
+ "additionalProperties": {
+ "type": "object",
+ "required": [
+ "label"
+ ],
+ "properties": {
+ "label": {
+ "type": "string",
+ "minLength": 1
+ },
+ "color": {
+ "type": "string"
+ },
+ "dimension": {
+ "type": "string",
+ "enum": [
+ "hero-radial",
+ "pairwise",
+ "both"
+ ]
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "relations": {
+ "type": "array",
+ "description": "Master edge list; each edge is stored exactly once. Stages reference edges via {a,b} optionally disambiguated by id/type; duplication is forbidden. When multiple edges share the same (a,b) pair (relationship evolves across stages), scope, type, or a stable relation.id must make references resolvable.",
+ "items": {
+ "type": "object",
+ "required": [
+ "a",
+ "b",
+ "type"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "(v1.3) Optional stable id for this master edge. When present, it is the preferred key for provenance and stage relation refs; duplicates are rejected (E12)."
+ },
+ "a": {
+ "type": "string",
+ "description": "Entity id (or a crawled name from aliases, normalized during validation)"
+ },
+ "b": {
+ "type": "string"
+ },
+ "type": {
+ "type": "string",
+ "description": "Must be registered in relationTypes"
+ },
+ "label": {
+ "type": "string"
+ },
+ "scope": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "minItems": 1,
+ "description": "(v1.1) This edge is only active in the listed stages (values are stages[].key). Without scope, the edge is active in all stages"
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "stages": {
+ "type": "array",
+ "description": "Stage/scene/event views. Carry only id references and stage-specific overlays; embedding entity fields is forbidden.",
+ "items": {
+ "type": "object",
+ "required": [
+ "key",
+ "name",
+ "entities"
+ ],
+ "properties": {
+ "key": {
+ "type": "string",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "desc": {
+ "type": "string"
+ },
+ "descMobile": {
+ "type": "string"
+ },
+ "entities": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "Subset of entity ids for this stage"
+ },
+ "layout": {
+ "type": "object",
+ "description": "Normalized coordinates keyed by entity id",
+ "additionalProperties": {
+ "type": "array",
+ "prefixItems": [
+ {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1
+ },
+ {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1
+ }
+ ],
+ "minItems": 2,
+ "maxItems": 2
+ }
+ },
+ "relations": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "a",
+ "b"
+ ],
+ "properties": {
+ "a": {
+ "type": "string"
+ },
+ "b": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "(v1.3) Optional: resolve this ref to the master edge whose relation.id matches."
+ },
+ "type": {
+ "type": "string",
+ "description": "(v1.3) Optional: disambiguate by edge type when multiple edges share (a,b)."
+ }
+ },
+ "additionalProperties": false
+ },
+ "description": "{a,b} keys referencing master relations; type/label come from the master edge"
+ },
+ "overlay": {
+ "type": "object",
+ "description": "Stage-specific information (persona/description overrides), keyed by entity id",
+ "additionalProperties": {
+ "type": "object"
+ }
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "attributeTypes": {
+ "type": "object",
+ "description": "(v1.1) Attribute-label registry (symmetric to relationTypes). Labels in the attribute-pair collections declared by attributeSources must be registered here.",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "label": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "attributeSources": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "(v1.1) Path declarations of attribute-pair collections; supports 'entities.*.' and 'domain.'. Every [label, value, ...] pair or {label, ...} object in the declared collections has its label checked by E10"
+ },
+ "contents": {
+ "type": "object",
+ "description": "(v1.1) Long-form content data layer: prose/timeline lifted out of template HTML. body is a restricted HTML subset; highlights declare in-text entity/annotation references (checked by E13 — a wrong entity name in prose is caught).",
+ "additionalProperties": {
+ "type": "object",
+ "required": [
+ "kind"
+ ],
+ "properties": {
+ "kind": {
+ "type": "string",
+ "description": "Content kind: timeline-item / article-section / ..."
+ },
+ "title": {
+ "type": "string"
+ },
+ "body": {
+ "type": "string",
+ "description": "Restricted-HTML-subset string"
+ },
+ "bodyFormat": {
+ "type": "string",
+ "enum": [
+ "html-subset",
+ "text"
+ ]
+ },
+ "highlights": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "ref"
+ ],
+ "properties": {
+ "text": {
+ "type": "string"
+ },
+ "ref": {
+ "type": "string",
+ "description": "Entity id / alias / domain.notes annotation key"
+ }
+ },
+ "additionalProperties": true
+ }
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "domain": {
+ "type": "object",
+ "description": "Library-specific data collections (works/questions/cast/notes, etc.), constrained by each library's own data.schema.json"
+ },
+ "assets": {
+ "type": "object",
+ "description": "Asset manifest; keys are asset paths as they appear in the data",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "sourceUrl": {
+ "type": "string"
+ },
+ "hash": {
+ "type": "string"
+ },
+ "exists": {
+ "type": "boolean"
+ },
+ "bytes": {
+ "type": "integer"
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "derivations": {
+ "type": "object",
+ "description": "v1.3 派生契约:声明哪些呈现是从 pack 数据派生的、怎么派生、谁消费。文档化契约(非可执行代码),只建模非平凡派生(重复/排序/重建/消解/投影)。",
+ "additionalProperties": {
+ "type": "object",
+ "required": [
+ "kind",
+ "source",
+ "consumers",
+ "note"
+ ],
+ "properties": {
+ "kind": {
+ "enum": [
+ "repeat",
+ "insertion-order",
+ "lookup-rebuild",
+ "scope-resolution",
+ "projection",
+ "reference-only"
+ ],
+ "description": "派生类型:repeat=集合重复渲染;insertion-order=键插入序即呈现序;lookup-rebuild=按字段反查重建;scope-resolution=方向/作用域消解;projection=字段直投但有重组;reference-only=仅存引用运行时重建"
+ },
+ "source": {
+ "type": "string",
+ "description": "派生源的 pack 内路径(如 domain.carouselTrack.entityIds / contents / entities.*.name)"
+ },
+ "consumers": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "消费方标识:引擎函数名或组件文件"
+ },
+ "affects": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "(可选)受影响的呈现区域/选择器"
+ },
+ "note": {
+ "type": "string",
+ "description": "人读说明:派生怎么工作、变更时的影响面"
+ },
+ "alsoTouches": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "(可选)次级影响源:除 source 外,变更这些路径也会影响此派生。用于 source 描述组合关系、但实体字段变更也影响呈现的情况(如 carouselTrack.source=domain.carouselTrack.entityIds,但 milestone.title 变更也影响渲染)。"
+ }
+ }
+ }
+ }
+ },
+ "$defs": {
+ "provEntry": {
+ "type": "object",
+ "properties": {
+ "origin": {
+ "type": "string",
+ "description": "Data origin: engine-embedded-defaults / example-json-contract / template-html-contents / crawl: etc."
+ },
+ "sourceUrl": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "Original HTTP(S) source URL (mandatory for crawl pipelines); runtime E15 validates protocol and hostname"
+ },
+ "fetchedAt": {
+ "type": "string",
+ "description": "Fetch date, YYYY-MM-DD"
+ },
+ "confidence": {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1,
+ "description": "Confidence; baseline data (original case page) is 1.0, crawled data gets actual values from the pipeline"
+ },
+ "note": {
+ "type": "string"
+ },
+ "fieldOrigins": {
+ "type": "object",
+ "description": "Optional field-level provenance for direct entity fields; keys are entity field names and values use the same provenance entry contract.",
+ "additionalProperties": {
+ "$ref": "#/$defs/provEntry"
+ }
+ }
+ },
+ "additionalProperties": true
+ }
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/experiment.schema.json b/research/agent-eval/results/executed-tooling/scripts/lib/experiment.schema.json
new file mode 100644
index 0000000..9146ecf
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/experiment.schema.json
@@ -0,0 +1,32 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://sg.local/contracts/agent-experiment-v1.json",
+ "title": "SG Agent ExperimentSpec v1.0",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["experimentVersion", "experimentId", "title", "agent", "tasks", "repetitions", "seed"],
+ "properties": {
+ "experimentVersion": { "const": "1.0" },
+ "experimentId": { "type": "string", "pattern": "^experiment:[0-9a-f]{64}$" },
+ "title": { "type": "string", "minLength": 1 },
+ "agent": {
+ "type": "object", "additionalProperties": false,
+ "required": ["kind", "provider", "model", "argv", "adapter"],
+ "properties": {
+ "kind": { "enum": ["scripted", "ai"] },
+ "provider": { "type": "string", "minLength": 1 },
+ "model": { "type": "string", "minLength": 1 },
+ "argv": { "type": "array", "minItems": 1, "items": { "type": "string" } },
+ "adapter": { "type": "object", "additionalProperties": false, "required": ["path", "sha256"], "properties": { "path": { "type": "string", "minLength": 1 }, "sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } } },
+ "infraExitCodes": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "integer", "minimum": 1, "maximum": 255 } }
+
+ }
+ },
+ "tasks": {
+ "type": "array", "minItems": 1,
+ "items": { "type": "object", "additionalProperties": false, "required": ["id", "manifest", "taskId", "manifestSha256"], "properties": { "id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,63}$" }, "manifest": { "type": "string", "minLength": 1 }, "taskId": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, "manifestSha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } } }
+ },
+ "repetitions": { "type": "integer", "minimum": 1, "maximum": 100 },
+ "seed": { "type": ["integer", "string"] }
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/run-report.schema.json b/research/agent-eval/results/executed-tooling/scripts/lib/run-report.schema.json
new file mode 100644
index 0000000..9907e9c
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/run-report.schema.json
@@ -0,0 +1,97 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://sg.local/run-report.schema.json",
+ "title": "SG Data Pack Run Report v1.0",
+ "type": "object",
+ "required": [
+ "reportVersion",
+ "run",
+ "inputs",
+ "summary",
+ "inventory",
+ "findings",
+ "changes",
+ "assurances",
+ "coverage",
+ "risks",
+ "nextSteps",
+ "operations",
+ "impacts",
+ "artifacts",
+ "raw"
+ ],
+ "additionalProperties": false,
+ "properties": {
+ "reportVersion": { "const": "1.0" },
+ "run": {
+ "type": "object",
+ "required": ["id", "command", "exitCode", "libId", "mode", "outcome", "maturity"],
+ "additionalProperties": false,
+ "properties": {
+ "id": { "type": "string", "pattern": "^sha256:" },
+ "command": { "type": "string", "minLength": 1 },
+ "exitCode": { "type": "integer", "enum": [0, 1, 2] },
+ "libId": { "type": "string" },
+ "mode": { "enum": ["pack", "integration", "evolution", "review", "candidate"] },
+ "outcome": { "enum": ["ready", "issues-found", "review-required", "blocked", "input-error"] },
+ "maturity": { "enum": ["UNASSESSED", "ASSESSED", "DATA-VALID"] }
+ }
+ },
+ "inputs": { "type": "object", "additionalProperties": true },
+ "summary": {
+ "type": "object",
+ "required": ["findings", "errors", "warnings", "open", "resolved", "notAssessed", "changes", "nextSteps"],
+ "additionalProperties": false,
+ "properties": {
+ "findings": { "type": "integer", "minimum": 0 },
+ "errors": { "type": "integer", "minimum": 0 },
+ "warnings": { "type": "integer", "minimum": 0 },
+ "open": { "type": "integer", "minimum": 0 },
+ "resolved": { "type": "integer", "minimum": 0 },
+ "notAssessed": { "type": "integer", "minimum": 0 },
+ "changes": { "type": "integer", "minimum": 0 },
+ "nextSteps": { "type": "integer", "minimum": 0 }
+ }
+ },
+ "inventory": { "type": "object", "additionalProperties": true },
+ "findings": { "type": "array", "items": { "$ref": "#/$defs/finding" } },
+ "changes": { "type": "array", "items": { "type": "object", "additionalProperties": true } },
+ "assurances": { "type": "array", "items": { "$ref": "#/$defs/assurance" } },
+ "coverage": { "type": "array", "items": { "$ref": "#/$defs/assurance" } },
+ "risks": { "type": "array", "items": { "type": "object", "additionalProperties": true } },
+ "nextSteps": { "type": "array", "items": { "type": "object", "additionalProperties": true } },
+ "operations": { "type": "array", "items": { "type": "object", "additionalProperties": true } },
+ "impacts": { "type": "array", "items": { "type": "object", "additionalProperties": true } },
+ "artifacts": { "type": "array", "items": { "type": "object", "additionalProperties": true } },
+ "raw": { "type": "object", "additionalProperties": true }
+ },
+ "$defs": {
+ "finding": {
+ "type": "object",
+ "required": ["id", "code", "severity", "status", "category", "phase", "title", "message", "blocking"],
+ "additionalProperties": true,
+ "properties": {
+ "id": { "type": "string" },
+ "code": { "type": "string" },
+ "severity": { "enum": ["error", "warning", "info"] },
+ "status": { "enum": ["open", "resolved", "review-required", "accepted-risk", "not-assessed"] },
+ "category": { "type": "string" },
+ "phase": { "type": "string" },
+ "title": { "type": "string" },
+ "message": { "type": "string" },
+ "blocking": { "type": "boolean" }
+ }
+ },
+ "assurance": {
+ "type": "object",
+ "required": ["id", "title", "status", "blocking"],
+ "additionalProperties": true,
+ "properties": {
+ "id": { "type": "string" },
+ "title": { "type": "string" },
+ "status": { "enum": ["passed", "failed", "not-assessed", "not-applicable", "review-required"] },
+ "blocking": { "type": "boolean" }
+ }
+ }
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-command-runner.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-command-runner.js
new file mode 100644
index 0000000..ecc49d7
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-command-runner.js
@@ -0,0 +1,159 @@
+'use strict';
+
+/* Synchronous, shell-free command execution with bounded inputs and outputs. */
+const fs = require('node:fs');
+const { spawnSync } = require('node:child_process');
+const { assertWithinRoot, canonicalPath } = require('./sg-path-policy.js');
+
+const DEFAULT_TIMEOUT = 30_000;
+const DEFAULT_MAX_BUFFER = 1024 * 1024;
+const DEFAULT_ENV_ALLOWLIST = [
+ 'PATH',
+ 'HOME',
+ 'TMPDIR',
+ 'TMP',
+ 'TEMP',
+ 'LANG',
+ 'LC_ALL',
+ 'LC_CTYPE',
+ 'SYSTEMROOT',
+ 'SystemRoot',
+ 'COMSPEC',
+ 'ComSpec',
+ 'PATHEXT',
+];
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+function assertArgv(argv) {
+ if (!Array.isArray(argv) || argv.length === 0) {
+ throw new TypeError('argv must be a non-empty array');
+ }
+ for (const argument of argv) {
+ if (typeof argument !== 'string') throw new TypeError('every argv item must be a string');
+ if (argument.includes('\0')) throw new TypeError('argv items must not contain null bytes');
+ }
+ if (argv[0].length === 0) throw new TypeError('argv[0] must name a command');
+}
+
+function normalizeAllowlist(value) {
+ if (value === undefined) return DEFAULT_ENV_ALLOWLIST;
+ if (value instanceof Set) value = [...value];
+ if (!Array.isArray(value) || value.some((name) => typeof name !== 'string' || !ENV_NAME.test(name))) {
+ throw new TypeError('envAllowlist must be an array or Set of environment variable names');
+ }
+ return value;
+}
+
+function allowedEnvironment(options = {}) {
+ const allowlist = normalizeAllowlist(options.envAllowlist === undefined ? options.allowedEnv : options.envAllowlist);
+ const supplied = options.env === undefined ? {} : options.env;
+ if (!supplied || typeof supplied !== 'object' || Array.isArray(supplied)) throw new TypeError('env must be an object');
+ const source = { ...process.env, ...supplied };
+ const output = Object.create(null);
+ for (const name of allowlist) {
+ if (!Object.prototype.hasOwnProperty.call(source, name) || source[name] === undefined) continue;
+ const value = source[name];
+ if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean') {
+ throw new TypeError(`environment variable ${name} must be a string, number, or boolean`);
+ }
+ const text = String(value);
+ if (text.includes('\0')) throw new TypeError(`environment variable ${name} must not contain null bytes`);
+ output[name] = text;
+ }
+ return output;
+}
+
+function positiveInteger(value, fallback, name) {
+ if (value === undefined) return fallback;
+ if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${name} must be a positive safe integer`);
+ return value;
+}
+
+function outputBuffer(value) {
+ if (Buffer.isBuffer(value)) return value;
+ if (value === null || value === undefined) return Buffer.alloc(0);
+ return Buffer.from(String(value));
+}
+
+function boundedText(value, maxBuffer) {
+ const bytes = outputBuffer(value);
+ const truncated = bytes.length > maxBuffer;
+ return {
+ text: bytes.subarray(0, maxBuffer).toString('utf8'),
+ truncated,
+ };
+}
+
+function structuredError(error) {
+ if (!error) return undefined;
+ return {
+ name: error.name,
+ code: error.code || null,
+ message: error.message,
+ errno: error.errno === undefined ? null : error.errno,
+ syscall: error.syscall || null,
+ path: error.path || null,
+ };
+}
+
+function runCommand(input, legacyOptions = {}) {
+ const objectForm = !Array.isArray(input) && input !== null && typeof input === 'object';
+ const argv = objectForm ? input.argv : input;
+ const options = objectForm ? input : legacyOptions;
+ assertArgv(argv);
+ if (!options || typeof options !== 'object' || Array.isArray(options)) throw new TypeError('options must be an object');
+ const allowedRoot = canonicalPath(options.allowedRoot || options.root || process.cwd(), { mustExist: true });
+ const cwd = assertWithinRoot(allowedRoot, options.cwd || allowedRoot, { mustExist: true });
+ if (!fs.statSync(cwd).isDirectory()) throw new TypeError(`cwd must be a directory: ${cwd}`);
+ const timeout = positiveInteger(options.timeoutMs === undefined ? options.timeout : options.timeoutMs, DEFAULT_TIMEOUT, 'timeoutMs');
+ const maxBuffer = positiveInteger(options.maxOutputBytes === undefined ? options.maxBuffer : options.maxOutputBytes, DEFAULT_MAX_BUFFER, 'maxOutputBytes');
+ const env = allowedEnvironment(options);
+ const started = process.hrtime.bigint();
+ const result = spawnSync(argv[0], argv.slice(1), {
+ cwd,
+ env,
+ input: options.input,
+ encoding: null,
+ timeout,
+ maxBuffer,
+ killSignal: options.killSignal || 'SIGTERM',
+ windowsHide: true,
+ shell: false,
+ });
+ const durationMs = Number(process.hrtime.bigint() - started) / 1e6;
+ const stdout = boundedText(result.stdout, maxBuffer);
+ const stderr = boundedText(result.stderr, maxBuffer);
+ const errorCode = result.error && result.error.code;
+ const timedOut = errorCode === 'ETIMEDOUT';
+ const outputLimitReached = errorCode === 'ENOBUFS';
+ const stdoutTruncated = stdout.truncated;
+ const stderrTruncated = stderr.truncated;
+
+ return {
+ argv: [...argv],
+ cwd,
+ status: result.status,
+ exitCode: result.status,
+ exit: result.status,
+ signal: result.signal,
+ timedOut,
+ durationMs,
+ duration: durationMs,
+ stdout: stdout.text,
+ stderr: stderr.text,
+ stdoutTruncated,
+ stderrTruncated,
+ truncated: outputLimitReached || stdoutTruncated || stderrTruncated,
+ error: structuredError(result.error),
+ };
+}
+
+module.exports = {
+ DEFAULT_TIMEOUT,
+ DEFAULT_MAX_BUFFER,
+ DEFAULT_ENV_ALLOWLIST,
+ allowedEnvironment,
+ buildAllowedEnv: allowedEnvironment,
+ runCommand,
+ run: runCommand,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-data-loader.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-data-loader.js
new file mode 100644
index 0000000..9465244
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-data-loader.js
@@ -0,0 +1,677 @@
+/*
+ * sg-data-loader.js — SG Data Pack runtime validator (zero-dependency UMD) v1.3
+ *
+ * Responsibilities:
+ * 1. validate(pack) -> { errors: [], warnings: [] } structural Data Pack validation
+ * 2. assertValid(pack) -> throws an aggregated error on failure (called by engines on mount — fails loudly, never silently)
+ * 3. resolveAlias(pack, name) / resolveId(pack, idOrName) alias normalization (crawled name -> canonical id)
+ *
+ * Validation rules (mirrors data-pack.schema.json):
+ * E1 schemaVersion must be "1.0", "1.1", "1.2", or "1.3"
+ * E2 meta.id / meta.title must be non-empty strings
+ * E3 entities must be an object; each entity needs a name (or the kind's nameField) and a kind
+ * E4 alias targets must exist in entities; alias keys must not collide with entity ids
+ * E5 relations[].a/b must exist in entities after alias normalization (dangling ref = error)
+ * E6 relations[].type must be registered in relationTypes (illegal enum = error)
+ * E7 references in stages[].entities/layout/overlay/relations must exist
+ * E8 layout coordinates must be within [0,1]
+ * E9 (reserved: alias disambiguation context)
+ * E10 attribute-pair labels in attributeSources collections must be registered in attributeTypes (v1.1)
+ * E11 local assets referenced by entities/contents/domain must be registered in the assets manifest (v1.1)
+ * E12 relation ids/scopes must be valid; ambiguous stage edge refs must resolve via id/type/scope (v1.1)
+ * E13 contents[].highlights[].ref must resolve to an entity/alias or domain.notes key (v1.1)
+ * E14 both sides of a sameAs pair must exist (v1.2)
+ * E15 provenance keys must point at existing entities/relations/contents (v1.2)
+ * E16 derivations entries: valid kind enum + source path resolvable within the pack (v1.3)
+ * W1 entity never referenced by any stage/relation
+ * W2 asset marked exists:false
+ * W3 relation missing label
+ * W4 local asset exists but lacks a hash (incomplete provenance)
+ * W5 provenance confidence below threshold (default 0.7, tunable via meta.confidenceThreshold) (v1.2)
+ * W6 entity names collide after normalization without a sameAs declaration (suspected duplicate, v1.2)
+ * W7 provenance entry missing origin (v1.3)
+ * W8 crawl provenance entry missing sourceUrl (v1.3)
+ */
+(function (global) {
+ 'use strict';
+
+ var SCHEMA_VERSIONS = ['1.0', '1.1', '1.2', '1.3'];
+ var IMG_EXT = /\.(png|jpe?g|webp|gif|svg)$/i;
+
+ function isObj(v) { return v !== null && typeof v === 'object' && !Array.isArray(v); }
+
+ function isHttpUrl(v) {
+ if (typeof v !== 'string' || /[\u0000-\u001f\u007f]/.test(v)) return false;
+ try {
+ var u = new URL(v);
+ return (u.protocol === 'http:' || u.protocol === 'https:') && !!u.hostname;
+ } catch (_) {
+ return false;
+ }
+ }
+
+ function resolveAlias(pack, name) {
+ if (pack && isObj(pack.aliases) && Object.prototype.hasOwnProperty.call(pack.aliases, name)) {
+ return pack.aliases[name];
+ }
+ return null;
+ }
+
+ /* Select one authoritative display field/value for an entity. E3, W1, and
+ * W6 must agree on this lookup: kindNameFields[kind], otherwise "name".
+ * Deliberately do not fall back to legacy name/title fields, because doing
+ * so can validate or compare a value that the entity kind does not display. */
+ function entityDisplay(pack, entity) {
+ var nameFields = pack && isObj(pack.kindNameFields) ? pack.kindNameFields : {};
+ var configured = isObj(entity) && typeof entity.kind === 'string' &&
+ Object.prototype.hasOwnProperty.call(nameFields, entity.kind)
+ ? nameFields[entity.kind]
+ : undefined;
+ var field = typeof configured === 'string' && configured ? configured : 'name';
+ return { field: field, value: isObj(entity) ? entity[field] : undefined };
+ }
+
+ /* crawled name / alias / canonical id -> canonical id; null when unresolvable.
+ * Handles the {id, context} disambiguation form: resolveAlias may return an
+ * object (E4 validates it), so we unpack .id here before the entity lookup. */
+ function resolveId(pack, idOrName) {
+ if (!pack || !isObj(pack.entities)) return null;
+ if (Object.prototype.hasOwnProperty.call(pack.entities, idOrName)) return idOrName;
+ var via = resolveAlias(pack, idOrName);
+ if (isObj(via)) via = via.id;
+ if (via && Object.prototype.hasOwnProperty.call(pack.entities, via)) return via;
+ return null;
+ }
+
+ function relationIdentity(pack, relation) {
+ if (relation && typeof relation.id === 'string' && relation.id) return relation.id;
+ var a = resolveId(pack, relation && relation.a) || (relation && relation.a);
+ var b = resolveId(pack, relation && relation.b) || (relation && relation.b);
+ var scope = Array.isArray(relation && relation.scope) ? relation.scope.slice().sort().join('|') : '*';
+ return a + '::' + b + '::' + (relation && relation.type) + '::scope=' + scope;
+ }
+
+ /* contents highlight ref: entity/alias, or a domain.notes annotation key */
+ function resolveContentRef(pack, ref) {
+ if (resolveId(pack, ref)) return ref;
+ if (isObj(pack.domain) && isObj(pack.domain.notes) &&
+ Object.prototype.hasOwnProperty.call(pack.domain.notes, ref)) return ref;
+ return null;
+ }
+
+ /* Resolve attributeSources paths: supports "entities.*.relations" and "domain.facts" forms */
+ function resolveAttrSource(pack, pathExpr) {
+ var out = [];
+ var m;
+ if ((m = /^entities\.\*\.(\w+)$/.exec(pathExpr))) {
+ var field = m[1];
+ Object.keys(pack.entities || {}).forEach(function (id) {
+ var v = pack.entities[id][field];
+ if (Array.isArray(v)) out.push({ at: 'entities.' + id + '.' + field, pairs: v });
+ });
+ } else if ((m = /^domain\.(\w+(?:\.\w+)*)$/.exec(pathExpr))) {
+ var segs = m[1].split('.');
+ var v = pack.domain;
+ for (var i = 0; i < segs.length && v !== undefined; i++) v = v[segs[i]];
+ if (Array.isArray(v)) out.push({ at: 'domain.' + segs.join('.'), pairs: v });
+ else if (isObj(v)) Object.keys(v).forEach(function (k) {
+ if (Array.isArray(v[k])) out.push({ at: 'domain.' + segs.join('.') + '.' + k, pairs: v[k] });
+ });
+ }
+ return out;
+ }
+
+ /* Collect all local asset references in entities/contents/domain (value-shape heuristic).
+ * domain is free-form but commonly carries asset paths (e.g. domain.works[*].img),
+ * so it must be covered by E11 just like entities and contents. */
+ function collectAssetRefs(pack) {
+ var refs = [];
+ function walk(v) {
+ if (typeof v === 'string') {
+ if (IMG_EXT.test(v) && !/^https?:\/\//i.test(v) && !/^data:/.test(v)) refs.push(v);
+ } else if (Array.isArray(v)) v.forEach(walk);
+ else if (isObj(v)) Object.keys(v).forEach(function (k) { walk(v[k]); });
+ }
+ walk(pack.entities || {});
+ walk(pack.contents || {});
+ walk(pack.domain || {});
+ return refs;
+ }
+
+ function validate(pack) {
+ var errors = [];
+ var warnings = [];
+
+ if (!isObj(pack)) {
+ return { errors: ['pack must be an object'], warnings: warnings };
+ }
+
+ /* E1 */
+ if (SCHEMA_VERSIONS.indexOf(pack.schemaVersion) === -1) {
+ errors.push('E1: schemaVersion must be ' + SCHEMA_VERSIONS.map(function (s) { return '"' + s + '"'; }).join(' or ') +
+ ', got ' + JSON.stringify(pack.schemaVersion));
+ }
+
+ /* E2 */
+ if (!isObj(pack.meta) || typeof pack.meta.id !== 'string' || !pack.meta.id) {
+ errors.push('E2: meta.id must be a non-empty string');
+ }
+ if (!isObj(pack.meta) || typeof pack.meta.title !== 'string' || !pack.meta.title) {
+ errors.push('E2: meta.title must be a non-empty string');
+ }
+ if (isObj(pack.meta) && pack.meta.confidenceThreshold !== undefined &&
+ (typeof pack.meta.confidenceThreshold !== 'number' || !isFinite(pack.meta.confidenceThreshold) ||
+ pack.meta.confidenceThreshold < 0 || pack.meta.confidenceThreshold > 1)) {
+ errors.push('E2: meta.confidenceThreshold must be a number between 0 and 1');
+ }
+
+ /* E3 */
+ var entities = pack.entities;
+ if (!isObj(entities) || Object.keys(entities).length === 0) {
+ errors.push('E3: entities must be a non-empty object');
+ entities = {};
+ }
+ Object.keys(entities).forEach(function (id) {
+ var e = entities[id];
+ if (!/^[a-z][a-z0-9_-]*$/.test(id) || id === '__proto__' || id === 'prototype' || id === 'constructor') {
+ errors.push('E3: entities.' + id + ' must use a safe stable slug id');
+ }
+ if (!isObj(e)) { errors.push('E3: entities.' + id + ' must be an object'); return; }
+ var display = entityDisplay(pack, e);
+ if (typeof display.value !== 'string' || !display.value) {
+ errors.push('E3: entities.' + id + '.' + display.field + ' must be a non-empty string' + (display.field !== 'name' ? ' (nameField of kind:' + e.kind + ')' : ''));
+ }
+ if (typeof e.kind !== 'string' || !e.kind) errors.push('E3: entities.' + id + '.kind must be a non-empty string');
+ });
+
+ /* E4 */
+ var aliases = pack.aliases || {};
+ if (!isObj(aliases)) {
+ errors.push('E4: aliases must be an object');
+ aliases = {};
+ }
+ Object.keys(aliases).forEach(function (alias) {
+ if (alias === '__proto__' || alias === 'prototype' || alias === 'constructor') {
+ errors.push('E4: alias "' + alias + '" is a reserved object key');
+ }
+ if (Object.prototype.hasOwnProperty.call(entities, alias)) {
+ errors.push('E4: alias "' + alias + '" collides with an entity id');
+ }
+ var target = aliases[alias];
+ if (isObj(target)) target = target.id; // {id, context} disambiguation form (E9 reserved)
+ if (!Object.prototype.hasOwnProperty.call(entities, target)) {
+ errors.push('E4: alias "' + alias + '" points to non-existent entity "' + target + '"');
+ }
+ });
+
+ /* relationTypes registry */
+ var relationTypes = {};
+ if (pack.relationTypes !== undefined && !isObj(pack.relationTypes)) {
+ errors.push('E6: relationTypes must be an object');
+ } else if (isObj(pack.relationTypes)) {
+ relationTypes = pack.relationTypes;
+ Object.keys(relationTypes).forEach(function (type) {
+ var entry = relationTypes[type];
+ if (!isObj(entry)) {
+ errors.push('E6: relationTypes.' + type + ' must be an object');
+ } else if (typeof entry.label !== 'string' || !entry.label) {
+ errors.push('E6: relationTypes.' + type + '.label must be a non-empty string');
+ }
+ });
+ }
+
+ /* stages pre-scan (E12 needs the stage-key set) */
+ var stages = Array.isArray(pack.stages) ? pack.stages : [];
+ if (pack.stages !== undefined && !Array.isArray(pack.stages)) {
+ errors.push('E7: stages must be an array');
+ }
+ var stageKeys = Object.create(null);
+ stages.forEach(function (s) { if (isObj(s) && typeof s.key === 'string') stageKeys[s.key] = true; });
+
+ /* E5/E6/E12/W3 */
+ var relations = Array.isArray(pack.relations) ? pack.relations : [];
+ if (pack.relations !== undefined && !Array.isArray(pack.relations)) {
+ errors.push('E5: relations must be an array');
+ }
+ var referenced = Object.create(null);
+ var relationIds = Object.create(null);
+ var anonymousRelationIds = Object.create(null);
+ relations.forEach(function (r, i) {
+ if (!isObj(r)) { errors.push('E5: relations[' + i + '] must be an object'); return; }
+ if (r.id !== undefined) {
+ if (typeof r.id !== 'string' || !r.id) errors.push('E12: relations[' + i + '].id must be a non-empty string when present');
+ else if (relationIds[r.id]) errors.push('E12: relations[' + i + '].id "' + r.id + '" is duplicated');
+ else relationIds[r.id] = true;
+ } else {
+ var anonymousIdentity = relationIdentity(pack, r);
+ if (anonymousRelationIds[anonymousIdentity] !== undefined) {
+ errors.push('E12: relations[' + i + '] duplicated master relation identity "' + anonymousIdentity + '" (first declared at relations[' + anonymousRelationIds[anonymousIdentity] + '])');
+ } else {
+ anonymousRelationIds[anonymousIdentity] = i;
+ }
+ }
+ ['a', 'b'].forEach(function (end) {
+ var id = resolveId(pack, r[end]);
+ if (!id) errors.push('E5: relations[' + i + '].' + end + ' dangling reference "' + r[end] + '"');
+ else referenced[id] = true;
+ });
+ if (typeof r.type !== 'string' || !Object.prototype.hasOwnProperty.call(relationTypes, r.type)) {
+ errors.push('E6: relations[' + i + '].type "' + r.type + '" is not registered in relationTypes');
+ }
+ if (r.label === undefined || r.label === '') {
+ warnings.push('W3: relations[' + i + '] (' + r.a + ' -> ' + r.b + ') missing label');
+ }
+ /* E12: scope stage keys must exist */
+ if (r.scope !== undefined) {
+ if (!Array.isArray(r.scope) || !r.scope.length) {
+ errors.push('E12: relations[' + i + '].scope must be a non-empty array of stage keys');
+ } else r.scope.forEach(function (k) {
+ if (!stageKeys[k]) errors.push('E12: relations[' + i + '].scope references non-existent stage "' + k + '"');
+ });
+ }
+ });
+
+ /* E7/E8 + E12 reference-disambiguation */
+ stages.forEach(function (s, i) {
+ if (!isObj(s)) { errors.push('E7: stages[' + i + '] must be an object'); return; }
+ if (typeof s.key !== 'string' || !s.key) errors.push('E7: stages[' + i + '].key must be a non-empty string');
+ else {
+ var dup = stages.some(function (o, j) { return j !== i && isObj(o) && o.key === s.key; });
+ if (dup) errors.push('E7: stages[' + i + '].key "' + s.key + '" is duplicated');
+ }
+ if (typeof s.name !== 'string' || !s.name) errors.push('E7: stages[' + i + '].name must be a non-empty string');
+
+ var memberSet = Object.create(null);
+ (Array.isArray(s.entities) ? s.entities : []).forEach(function (id) {
+ var rid = resolveId(pack, id);
+ if (!rid) errors.push('E7: stages[' + i + '].entities dangling reference "' + id + '"');
+ else { memberSet[rid] = true; referenced[rid] = true; }
+ });
+
+ if (s.layout !== undefined) {
+ if (!isObj(s.layout)) errors.push('E7: stages[' + i + '].layout must be an object');
+ else Object.keys(s.layout).forEach(function (id) {
+ var rid = resolveId(pack, id);
+ if (!rid) { errors.push('E7: stages[' + i + '].layout dangling reference "' + id + '"'); return; }
+ if (!Object.prototype.hasOwnProperty.call(memberSet, rid)) errors.push('E7: stages[' + i + '].layout."' + id + '" is not in this stage\'s entities');
+ var xy = s.layout[id];
+ if (!Array.isArray(xy) || xy.length !== 2 || typeof xy[0] !== 'number' || typeof xy[1] !== 'number') {
+ errors.push('E8: stages[' + i + '].layout."' + id + '" must be a numeric [x, y] pair');
+ } else if (xy[0] < 0 || xy[0] > 1 || xy[1] < 0 || xy[1] > 1) {
+ errors.push('E8: stages[' + i + '].layout."' + id + '" coordinates out of range [' + xy + '] (must be within [0,1])');
+ }
+ });
+ }
+
+ if (s.relations !== undefined) {
+ if (!Array.isArray(s.relations)) errors.push('E7: stages[' + i + '].relations must be an array');
+ else s.relations.forEach(function (ref, j) {
+ if (!isObj(ref)) { errors.push('E7: stages[' + i + '].relations[' + j + '] must be an {a,b} reference'); return; }
+ var at = 'stages[' + i + '].relations[' + j + ']';
+ var okA = resolveId(pack, ref.a), okB = resolveId(pack, ref.b);
+ if (!okA) errors.push('E7: ' + at + '.a dangling reference "' + ref.a + '"');
+ if (!okB) errors.push('E7: ' + at + '.b dangling reference "' + ref.b + '"');
+ if (!okA || !okB) return;
+ /* E12: a stage ref must resolve to exactly one active master edge.
+ * Endpoints are canonicalized via resolveId; explicit id/type constraints always
+ * match, and scoped edges are active only in their declared stages. */
+ var ra = okA, rb = okB;
+ var endpointCands = relations.filter(function (r) {
+ return isObj(r) && resolveId(pack, r.a) === ra && resolveId(pack, r.b) === rb;
+ });
+ var cands = endpointCands;
+ var validRefId = true;
+ if (ref.id !== undefined) {
+ if (typeof ref.id !== 'string' || !ref.id) {
+ errors.push('E12: ' + at + '.id must be a non-empty string when present');
+ validRefId = false;
+ cands = [];
+ } else {
+ cands = cands.filter(function (r) { return r.id === ref.id; });
+ }
+ }
+ if (ref.type !== undefined) {
+ if (typeof ref.type !== 'string' || !ref.type) {
+ errors.push('E12: ' + at + '.type must be a non-empty string when present');
+ cands = [];
+ } else {
+ cands = cands.filter(function (r) { return r.type === ref.type; });
+ }
+ }
+ cands = cands.filter(function (r) {
+ return !Array.isArray(r.scope) || r.scope.indexOf(s.key) !== -1;
+ });
+ if (cands.length > 1) {
+ var scoped = cands.filter(function (r) { return Array.isArray(r.scope); });
+ if (scoped.length === 1) cands = scoped;
+ }
+ if (endpointCands.length === 0) {
+ errors.push('E7: ' + at + ' {' + ref.a + ',' + ref.b + '} does not exist in master relations');
+ } else if (cands.length === 0 && validRefId) {
+ errors.push('E12: ' + at + ' {' + ref.a + ',' + ref.b + '} matches no active master edge for the declared id/type/scope (stage "' + s.key + '")');
+ } else if (cands.length > 1) {
+ errors.push('E12: ' + at + ' {' + ref.a + ',' + ref.b + '} matches ' + cands.length +
+ ' active master edges and cannot be resolved via id/type/scope (stage "' + s.key + '")');
+ }
+ });
+ }
+
+ if (s.overlay !== undefined) {
+ if (!isObj(s.overlay)) errors.push('E7: stages[' + i + '].overlay must be an object');
+ else Object.keys(s.overlay).forEach(function (id) {
+ if (!resolveId(pack, id)) errors.push('E7: stages[' + i + '].overlay dangling reference "' + id + '"');
+ });
+ }
+ });
+
+ /* E10: attributeTypes / attributeSources */
+ if (pack.attributeTypes !== undefined && !isObj(pack.attributeTypes)) {
+ errors.push('E10: attributeTypes must be an object');
+ }
+ if (pack.attributeSources !== undefined) {
+ if (!Array.isArray(pack.attributeSources)) {
+ errors.push('E10: attributeSources must be an array of path strings');
+ } else if (!isObj(pack.attributeTypes) || Object.keys(pack.attributeTypes).length === 0) {
+ errors.push('E10: attributeSources declared but attributeTypes is empty');
+ } else {
+ pack.attributeSources.forEach(function (pathExpr) {
+ var sources = resolveAttrSource(pack, pathExpr);
+ if (!sources.length) {
+ warnings.push('E10: attributeSources path "' + pathExpr + '" matched no attribute-pair collection');
+ return;
+ }
+ sources.forEach(function (src) {
+ src.pairs.forEach(function (pair, i) {
+ var label = Array.isArray(pair) ? pair[0] : (isObj(pair) ? pair.label : undefined);
+ if (typeof label !== 'string' || !Object.prototype.hasOwnProperty.call(pack.attributeTypes, label)) {
+ errors.push('E10: ' + src.at + '[' + i + '] attribute label ' + JSON.stringify(label) + ' is not registered in attributeTypes');
+ }
+ });
+ });
+ });
+ }
+ }
+
+ /* E11: local asset references must be registered in the assets manifest */
+ if (pack.assets !== undefined && !isObj(pack.assets)) {
+ errors.push('assets must be an object');
+ }
+ var assets = isObj(pack.assets) ? pack.assets : {};
+ var assetBase = (isObj(pack.meta) && typeof pack.meta.assetBase === 'string') ? pack.meta.assetBase : '';
+ collectAssetRefs(pack).forEach(function (v) {
+ var key = assetBase && v.indexOf('assets/') === -1 ? assetBase + v : v;
+ if (!Object.prototype.hasOwnProperty.call(assets, key)) {
+ errors.push('E11: asset reference "' + v + '" (resolved as "' + key + '") is not registered in the assets manifest');
+ }
+ });
+
+ /* E13: contents highlight refs */
+ if (pack.contents !== undefined) {
+ if (!isObj(pack.contents)) {
+ errors.push('E13: contents must be an object');
+ } else Object.keys(pack.contents).forEach(function (cid) {
+ var c = pack.contents[cid];
+ if (!isObj(c)) { errors.push('E13: contents.' + cid + ' must be an object'); return; }
+ if (typeof c.kind !== 'string' || !c.kind) errors.push('E13: contents.' + cid + '.kind must be a non-empty string');
+ if (c.body !== undefined && typeof c.body !== 'string') errors.push('E13: contents.' + cid + '.body must be a string');
+ if (c.highlights !== undefined) {
+ if (!Array.isArray(c.highlights)) errors.push('E13: contents.' + cid + '.highlights must be an array');
+ else c.highlights.forEach(function (h, i) {
+ if (!isObj(h) || typeof h.ref !== 'string') {
+ errors.push('E13: contents.' + cid + '.highlights[' + i + '] must contain a ref string');
+ return;
+ }
+ if (!resolveContentRef(pack, h.ref)) {
+ errors.push('E13: contents.' + cid + '.highlights[' + i + '].ref "' + h.ref + '" cannot be resolved (not an entity/alias/annotation key)');
+ }
+ });
+ }
+ });
+ }
+
+ /* W1 */
+ Object.keys(entities).forEach(function (id) {
+ var selected = entityDisplay(pack, entities[id]);
+ var display = typeof selected.value === 'string' && selected.value ? selected.value : id;
+ if (!referenced[id]) warnings.push('W1: entity "' + id + '" (' + display + ') is not referenced by any relation/stage');
+ });
+
+ /* E14: sameAs entity-identity pairs (v1.2) */
+ var sameAsPairs = [];
+ if (pack.sameAs !== undefined) {
+ if (!Array.isArray(pack.sameAs)) {
+ errors.push('E14: sameAs must be an array of [idA, idB] pairs');
+ } else pack.sameAs.forEach(function (pair, i) {
+ if (!Array.isArray(pair) || pair.length !== 2) {
+ errors.push('E14: sameAs[' + i + '] must be a two-element [idA, idB] array');
+ return;
+ }
+ pair.forEach(function (id) {
+ if (!resolveId(pack, id)) errors.push('E14: sameAs[' + i + '] references non-existent entity "' + id + '"');
+ });
+ if (pair[0] === pair[1]) errors.push('E14: sameAs[' + i + '] has the same entity on both sides');
+ sameAsPairs.push(pair);
+ });
+ }
+
+ /* E15/W5: provenance record-level provenance (v1.2) */
+ if (pack.provenance !== undefined) {
+ if (!isObj(pack.provenance)) {
+ errors.push('E15: provenance must be an object');
+ } else {
+ var threshold = (isObj(pack.meta) && typeof pack.meta.confidenceThreshold === 'number' &&
+ isFinite(pack.meta.confidenceThreshold) && pack.meta.confidenceThreshold >= 0 && pack.meta.confidenceThreshold <= 1)
+ ? pack.meta.confidenceThreshold : 0.7;
+ var checkProv = function (group, exists, describe) {
+ var g = pack.provenance[group];
+ if (g === undefined) return;
+ if (!isObj(g)) { errors.push('E15: provenance.' + group + ' must be an object'); return; }
+ Object.keys(g).forEach(function (key) {
+ if (!exists(key)) { errors.push('E15: provenance.' + group + '."' + key + '" ' + describe); return; }
+ var p = g[key];
+ if (!isObj(p)) {
+ errors.push('E15: ' + group + '."' + key + '" must be a provenance object');
+ return;
+ }
+ if (typeof p.origin !== 'string' || !p.origin) {
+ warnings.push('W7: ' + group + '."' + key + '" missing origin (provenance should record where the data came from)');
+ }
+ if (p.confidence !== undefined &&
+ (typeof p.confidence !== 'number' || !isFinite(p.confidence) || p.confidence < 0 || p.confidence > 1)) {
+ errors.push('E15: ' + group + '."' + key + '" confidence must be a finite number between 0 and 1');
+ } else if (typeof p.confidence === 'number' && p.confidence < threshold) {
+ warnings.push('W5: ' + group + '."' + key + '" confidence=' + p.confidence + ' is below threshold ' + threshold + ' (low-confidence data; manual review advised)');
+ }
+ if (typeof p.origin === 'string' && p.origin.indexOf('crawl:') === 0 &&
+ (p.sourceUrl === undefined || p.sourceUrl === null || p.sourceUrl === '')) {
+ warnings.push('W8: ' + group + '."' + key + '" origin is crawl:* but sourceUrl is missing');
+ } else if (p.sourceUrl !== undefined && p.sourceUrl !== null && p.sourceUrl !== '' && !isHttpUrl(p.sourceUrl)) {
+ errors.push('E15: ' + group + '."' + key + '" sourceUrl must be a valid HTTP(S) URL');
+ }
+ if (group === 'entities' && isObj(p)) {
+ if (p.fieldOrigins !== undefined && !isObj(p.fieldOrigins)) {
+ errors.push('E15: provenance.entities."' + key + '".fieldOrigins must be an object');
+ } else if (isObj(p.fieldOrigins)) {
+ Object.keys(p.fieldOrigins).forEach(function (field) {
+ if (!isObj(entities[key]) || !Object.prototype.hasOwnProperty.call(entities[key], field)) {
+ errors.push('E15: provenance.entities."' + key + '".fieldOrigins."' + field + '" points to a non-existent entity field');
+ return;
+ }
+ var fp = p.fieldOrigins[field];
+ if (!isObj(fp)) {
+ errors.push('E15: entities."' + key + '".fieldOrigins."' + field + '" must be a provenance object');
+ return;
+ }
+ if (typeof fp.origin !== 'string' || !fp.origin) {
+ warnings.push('W7: entities."' + key + '".fieldOrigins."' + field + '" missing origin (provenance should record where the data came from)');
+ }
+ if (fp.confidence !== undefined &&
+ (typeof fp.confidence !== 'number' || !isFinite(fp.confidence) || fp.confidence < 0 || fp.confidence > 1)) {
+ errors.push('E15: entities."' + key + '".fieldOrigins."' + field + '" confidence must be a finite number between 0 and 1');
+ } else if (typeof fp.confidence === 'number' && fp.confidence < threshold) {
+ warnings.push('W5: entities."' + key + '".fieldOrigins."' + field + '" confidence=' + fp.confidence + ' is below threshold ' + threshold + ' (low-confidence data; manual review advised)');
+ }
+ if (typeof fp.origin === 'string' && fp.origin.indexOf('crawl:') === 0 &&
+ (fp.sourceUrl === undefined || fp.sourceUrl === null || fp.sourceUrl === '')) {
+ warnings.push('W8: entities."' + key + '".fieldOrigins."' + field + '" origin is crawl:* but sourceUrl is missing');
+ } else if (fp.sourceUrl !== undefined && fp.sourceUrl !== null && fp.sourceUrl !== '' && !isHttpUrl(fp.sourceUrl)) {
+ errors.push('E15: entities."' + key + '".fieldOrigins."' + field + '" sourceUrl must be a valid HTTP(S) URL');
+ }
+ });
+ }
+ }
+ });
+ };
+ checkProv('entities', function (id) { return Object.prototype.hasOwnProperty.call(entities, id); }, 'points to a non-existent entity');
+ checkProv('relations', function (key) {
+ if (relations.some(function (r) { return isObj(r) && (relationIdentity(pack, r) === key || r.id === key); })) return true;
+ // v1.2 compatibility: a::b is accepted only when it identifies one
+ // canonical pair unambiguously. New packs should use relation.id or
+ // the full a::b::type::scope=<...> identity.
+ var pairMatches = relations.filter(function (r) {
+ if (!isObj(r)) return false;
+ var a = resolveId(pack, r.a) || r.a;
+ var b = resolveId(pack, r.b) || r.b;
+ return (a + '::' + b) === key || (r.a + '::' + r.b) === key;
+ });
+ return pairMatches.length === 1;
+ }, 'points to a non-existent or ambiguous relation (use relation.id or a::b::type::scope=<...>)');
+ checkProv('contents', function (key) {
+ return isObj(pack.contents) && Object.prototype.hasOwnProperty.call(pack.contents, key);
+ }, 'points to a non-existent contents entry');
+
+ }
+ }
+
+ /* E16: derivations declaration consistency (v1.3) */
+ if (pack.derivations !== undefined) {
+ var DERIV_KINDS = ['repeat', 'insertion-order', 'lookup-rebuild', 'scope-resolution', 'projection', 'reference-only'];
+ var resolvePath = function (expr) {
+ // dotted path with optional '*' wildcard: walk pack sections
+ var segs = String(expr).split('.');
+ var roots = Object.create(null);
+ ['entities', 'aliases', 'relationTypes', 'heroRelTypes', 'relations', 'stages', 'attributeTypes', 'attributeSources', 'contents', 'domain', 'assets', 'kindNameFields', 'sameAs', 'provenance', 'derivations', 'meta'].forEach(function (root) { roots[root] = true; });
+ if (!Object.prototype.hasOwnProperty.call(roots, segs[0])) return { rootOk: false, resolved: false };
+ var nodes = [pack];
+ for (var i = 0; i < segs.length; i++) {
+ var next = [];
+ for (var j = 0; j < nodes.length; j++) {
+ var node = nodes[j];
+ if (!isObj(node) && !Array.isArray(node)) continue;
+ if (segs[i] === '*') {
+ Object.keys(node).forEach(function (k) { next.push(node[k]); });
+ } else if (Object.prototype.hasOwnProperty.call(node, segs[i])) {
+ next.push(node[segs[i]]);
+ }
+ }
+ nodes = next;
+ if (!nodes.length) return { rootOk: true, resolved: false };
+ }
+ return { rootOk: true, resolved: nodes.length > 0 };
+ };
+ if (!isObj(pack.derivations)) {
+ errors.push('E16: derivations must be an object');
+ } else {
+ Object.keys(pack.derivations).forEach(function (dk) {
+ var d = pack.derivations[dk];
+ var at = 'derivations.' + dk;
+ if (!isObj(d)) { errors.push('E16: ' + at + ' must be an object'); return; }
+ if (DERIV_KINDS.indexOf(d.kind) === -1) {
+ errors.push('E16: ' + at + '.kind ' + JSON.stringify(d.kind) + ' is not a valid derivation kind (' + DERIV_KINDS.join('/') + ')');
+ }
+ if (typeof d.source !== 'string' || !d.source) {
+ errors.push('E16: ' + at + '.source must be a non-empty pack path string');
+ } else {
+ var r = resolvePath(d.source);
+ if (!r.rootOk) errors.push('E16: ' + at + '.source "' + d.source + '" has an invalid root section');
+ else if (!r.resolved) warnings.push('E16: ' + at + '.source "' + d.source + '" resolves to nothing in this pack');
+ }
+ if (!Array.isArray(d.consumers) || !d.consumers.length || d.consumers.some(function (c) { return typeof c !== 'string' || !c; })) {
+ errors.push('E16: ' + at + '.consumers must be a non-empty array of strings');
+ }
+ if (typeof d.note !== 'string' || !d.note) {
+ errors.push('E16: ' + at + '.note must be a non-empty string');
+ }
+ /* alsoTouches is a secondary pack-source path and is resolved like
+ * source. affects, by contrast, names selectors/regions and is only
+ * required to be an array of non-empty strings. */
+ if (d.alsoTouches !== undefined) {
+ if (!Array.isArray(d.alsoTouches) || d.alsoTouches.some(function (x) { return typeof x !== 'string' || !x; })) {
+ errors.push('E16: ' + at + '.alsoTouches must be an array of non-empty pack path strings');
+ } else d.alsoTouches.forEach(function (expr) {
+ var rr = resolvePath(expr);
+ if (!rr.rootOk) errors.push('E16: ' + at + '.alsoTouches "' + expr + '" has an invalid root section');
+ else if (!rr.resolved) warnings.push('E16: ' + at + '.alsoTouches "' + expr + '" resolves to nothing in this pack');
+ });
+ }
+ if (d.affects !== undefined && (!Array.isArray(d.affects) || d.affects.some(function (x) { return typeof x !== 'string' || !x; }))) {
+ errors.push('E16: ' + at + '.affects must be an array of non-empty selector/region strings');
+ }
+ });
+ }
+ }
+
+ /* W6: entity names collide after normalization without sameAs (v1.2) */
+ var normName = function (s) {
+ return String(s == null ? '' : s).replace(/[\s·・()()《》〈〉\-—_]+/g, '').toLowerCase();
+ };
+ var byNorm = Object.create(null);
+ Object.keys(entities).forEach(function (id) {
+ var e = entities[id];
+ if (!isObj(e)) return;
+ var selected = entityDisplay(pack, e);
+ var n = normName(selected.value || '');
+ if (!n) return;
+ if (!byNorm[n]) byNorm[n] = [];
+ byNorm[n].push(id);
+ });
+ Object.keys(byNorm).forEach(function (n) {
+ var ids = byNorm[n];
+ if (ids.length < 2) return;
+ for (var i = 0; i < ids.length; i++) for (var j = i + 1; j < ids.length; j++) {
+ var linked = sameAsPairs.some(function (p) {
+ return (p[0] === ids[i] && p[1] === ids[j]) || (p[0] === ids[j] && p[1] === ids[i]);
+ });
+ if (!linked) {
+ warnings.push('W6: entities "' + ids[i] + '" and "' + ids[j] + '" have identical normalized names (' + n + '); suspected duplicates — declare sameAs if they are the same subject');
+ }
+ }
+ });
+
+ /* W2/W4 */
+ Object.keys(assets).forEach(function (p) {
+ var a = assets[p];
+ if (isObj(a) && a.exists === false) warnings.push('W2: missing asset ' + p);
+ if (isObj(a) && a.exists === true && !a.hash && !/^https?:\/\//i.test(p)) {
+ warnings.push('W4: asset missing hash (incomplete provenance) ' + p);
+ }
+ });
+
+ /* domain is free-form, governed by library-level domainChecks/schema */
+ if (pack.domain !== undefined && !isObj(pack.domain)) {
+ errors.push('domain must be an object');
+ }
+
+ return { errors: errors, warnings: warnings };
+ }
+
+ function assertValid(pack) {
+ var r = validate(pack);
+ if (r.errors.length) {
+ var id = (pack && pack.meta && pack.meta.id) || '(unknown)';
+ throw new Error('[SGDataLoader] Data Pack "' + id + '" validation failed (' + r.errors.length + ' error(s)):\n - ' + r.errors.join('\n - '));
+ }
+ return r;
+ }
+
+ global.SGDataLoader = {
+ SCHEMA_VERSIONS: SCHEMA_VERSIONS,
+ validate: validate,
+ assertValid: assertValid,
+ resolveAlias: resolveAlias,
+ resolveId: resolveId,
+ entityDisplay: entityDisplay,
+ resolveContentRef: resolveContentRef,
+ relationIdentity: relationIdentity
+ };
+})(typeof window !== 'undefined' ? window : globalThis);
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-data-surface-import.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-data-surface-import.js
new file mode 100644
index 0000000..82f369d
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-data-surface-import.js
@@ -0,0 +1,143 @@
+'use strict';
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+
+const HASH_PATTERN = /^[a-f0-9]{64}$/;
+/* Every root field declared by the Data Pack v1.3 contract is forbidden in a
+ * Data Surface handoff. Keep the legacy adapters entry too: accepting it would
+ * let an older Data Pack-shaped payload bypass the handoff boundary. */
+const DATA_PACK_FIELDS = [
+ 'meta', 'kindNameFields', 'sameAs', 'provenance',
+ 'entities', 'aliases', 'relationTypes', 'heroRelTypes', 'relations', 'stages',
+ 'attributeTypes', 'attributeSources', 'contents', 'domain', 'assets', 'derivations',
+ 'adapters',
+];
+
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function issue(issues, path, message) {
+ issues.push({ path, message });
+}
+
+function validateSurface(surface, index, issues) {
+ const path = `surfaces[${index}]`;
+ if (!isObject(surface)) {
+ issue(issues, path, 'surface must be an object');
+ return;
+ }
+ if (typeof surface.id !== 'string' || !surface.id) issue(issues, `${path}.id`, 'surface id is required');
+ if (!isObject(surface.owner) || typeof surface.owner.componentFile !== 'string') issue(issues, `${path}.owner`, 'component owner is required');
+ if (!isObject(surface.source)) issue(issues, `${path}.source`, 'source description is required');
+ if (isObject(surface.source?.static) && Object.prototype.hasOwnProperty.call(surface.source.static, 'value')) {
+ issue(issues, `${path}.source.static.value`, 'raw business values are not allowed in a Data Surface Manifest');
+ }
+ if (!isObject(surface.shape)) issue(issues, `${path}.shape`, 'shape is required');
+ if (!Array.isArray(surface.fields)) issue(issues, `${path}.fields`, 'fields must be an array');
+ if (!Array.isArray(surface.consumers)) issue(issues, `${path}.consumers`, 'consumers must be an array');
+ if (!Array.isArray(surface.references)) issue(issues, `${path}.references`, 'references must be an array');
+ if (!Array.isArray(surface.unresolved)) issue(issues, `${path}.unresolved`, 'unresolved must be an array');
+ if (Object.prototype.hasOwnProperty.call(surface, 'reviewRequired') && typeof surface.reviewRequired !== 'boolean') {
+ issue(issues, `${path}.reviewRequired`, 'reviewRequired must be a boolean when present');
+ }
+}
+
+function validateDataSurfaceManifest(manifest) {
+ const issues = [];
+ if (!isObject(manifest)) return { valid: false, issues: [{ path: '$', message: 'manifest must be an object' }] };
+ for (const field of DATA_PACK_FIELDS) {
+ if (Object.prototype.hasOwnProperty.call(manifest, field)) issue(issues, field, `Data Pack field ${field} must not be present in the handoff manifest`);
+ }
+ if (manifest.schemaVersion !== '1.0') issue(issues, 'schemaVersion', 'schemaVersion must be 1.0');
+ if (manifest.kind !== 'data-surface-manifest') issue(issues, 'kind', 'kind must be data-surface-manifest');
+ if (!isObject(manifest.identity)) issue(issues, 'identity', 'identity is required');
+ else {
+ if (manifest.identity.contractVersion !== '1.0') issue(issues, 'identity.contractVersion', 'contractVersion must be 1.0');
+ if (typeof manifest.identity.sourceRoot !== 'string' || !manifest.identity.sourceRoot) issue(issues, 'identity.sourceRoot', 'sourceRoot is required');
+ for (const field of ['sourceHash', 'fixtureHash', 'configurationHash']) {
+ if (!HASH_PATTERN.test(manifest.identity[field] || '')) issue(issues, `identity.${field}`, `${field} must be a SHA-256 digest`);
+ }
+ }
+ if (!isObject(manifest.library)) issue(issues, 'library', 'library description is required');
+ if (!Array.isArray(manifest.surfaces)) issue(issues, 'surfaces', 'surfaces must be an array');
+ else manifest.surfaces.forEach((surface, index) => validateSurface(surface, index, issues));
+ if (!Array.isArray(manifest.unresolved)) issue(issues, 'unresolved', 'unresolved must be an array');
+ if (!isObject(manifest.metrics)) issue(issues, 'metrics', 'metrics are required');
+ else if (Array.isArray(manifest.surfaces) && manifest.metrics.surfaces !== manifest.surfaces.length) issue(issues, 'metrics.surfaces', 'surface count does not match');
+ if (typeof manifest.reviewRequired !== 'boolean') issue(issues, 'reviewRequired', 'reviewRequired must be a boolean');
+ return { valid: issues.length === 0, issues };
+}
+
+function canonicalize(value) {
+ if (Array.isArray(value)) return value.map(canonicalize);
+ if (isObject(value)) return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonicalize(value[key])]));
+ return value;
+}
+
+function hashCanonical(value) {
+ return crypto.createHash('sha256').update(JSON.stringify(canonicalize(value))).digest('hex');
+}
+
+function importDataSurfaceManifest(manifest, options = {}) {
+ const validation = validateDataSurfaceManifest(manifest);
+ if (!validation.valid) {
+ const error = new Error(`invalid Data Surface Manifest: ${validation.issues.map(item => `${item.path}: ${item.message}`).join('; ')}`);
+ error.code = 'INVALID_DATA_SURFACE_MANIFEST';
+ error.issues = validation.issues;
+ throw error;
+ }
+ const surfaceBlockers = manifest.surfaces
+ .filter(surface => surface.reviewRequired || surface.unresolved.length > 0)
+ .map(surface => `surface ${surface.id} requires review`);
+ const blockers = [
+ ...(manifest.reviewRequired ? ['manifest reviewRequired=true'] : []),
+ ...(manifest.unresolved.length ? [`manifest has ${manifest.unresolved.length} unresolved evidence records`] : []),
+ ...surfaceBlockers,
+ ];
+ const reviewRequired = blockers.length > 0;
+ const report = {
+ schemaVersion: '1.0',
+ kind: 'data-surface-import-report',
+ status: reviewRequired ? 'review-required' : 'ready',
+ source: {
+ kind: manifest.kind,
+ identity: manifest.identity,
+ digest: hashCanonical(manifest),
+ },
+ library: manifest.library,
+ surfaces: manifest.surfaces.map(surface => ({
+ id: surface.id,
+ owner: surface.owner,
+ source: surface.source,
+ shape: surface.shape,
+ fields: surface.fields,
+ consumers: surface.consumers,
+ injection: surface.injection,
+ references: surface.references,
+ evidence: surface.evidence,
+ reviewRequired: surface.reviewRequired,
+ unresolved: surface.unresolved,
+ })),
+ unresolved: manifest.unresolved,
+ review: manifest.review || { blockers: manifest.unresolved, policyNotices: [] },
+ metrics: manifest.metrics,
+ blockers,
+ reviewRequired,
+ dataPackGenerationAllowed: !reviewRequired,
+ };
+ if (reviewRequired && options.requireReady) {
+ const error = new Error(`Data Surface Manifest requires review: ${blockers.join('; ')}`);
+ error.code = 'DATA_SURFACE_REVIEW_REQUIRED';
+ error.report = report;
+ throw error;
+ }
+ return report;
+}
+
+function readDataSurfaceManifest(file, options) {
+ return importDataSurfaceManifest(JSON.parse(fs.readFileSync(file, 'utf8')), options);
+}
+
+module.exports = { DATA_PACK_FIELDS, validateDataSurfaceManifest, importDataSurfaceManifest, readDataSurfaceManifest };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-evidence-utils.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-evidence-utils.js
new file mode 100644
index 0000000..eac0da3
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-evidence-utils.js
@@ -0,0 +1,96 @@
+'use strict';
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+
+function stableValue(value) {
+ if (Array.isArray(value)) return value.map(stableValue);
+ if (!value || typeof value !== 'object') return value;
+ return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])]));
+}
+
+function stableJson(value) {
+ return JSON.stringify(stableValue(value));
+}
+
+function sha256Bytes(value) {
+ return 'sha256:' + crypto.createHash('sha256').update(value).digest('hex');
+}
+
+function contentId(prefix, value, omitted = []) {
+ const copy = { ...value };
+ for (const key of omitted) delete copy[key];
+ return `${prefix}:${sha256Bytes(stableJson(copy)).slice(7)}`;
+}
+
+function isDigest(value) {
+ return typeof value === 'string' && /^sha256:[0-9a-f]{64}$/.test(value);
+}
+
+function safeRelative(value, label = 'path') {
+ if (typeof value !== 'string' || !value || /[\u0000-\u001f\u007f]/.test(value)) throw new Error(`${label} must be a non-empty safe relative path`);
+ if (path.isAbsolute(value) || /^[A-Za-z]:[\\/]/.test(value)) throw new Error(`${label} must be relative`);
+ const normalized = value.replace(/\\/g, '/');
+ if (normalized.split('/').includes('..')) throw new Error(`${label} must not escape its root`);
+ return normalized.replace(/^\.\//, '');
+}
+
+function verifyArtifacts(artifacts, baseDir, errors) {
+ const verifiedIds = new Set();
+ const ids = new Set();
+ const paths = new Set();
+ let root = null;
+ if ((artifacts || []).length && !baseDir) errors.push('artifact verification baseDir is required');
+ else if (baseDir) {
+ try { root = fs.realpathSync(baseDir); }
+ catch (error) { errors.push(`artifact verification root cannot be resolved: ${error.message}`); }
+ }
+ for (const artifact of artifacts || []) {
+ if (!artifact || typeof artifact !== 'object' || Array.isArray(artifact)) {
+ errors.push('artifact must be an object');
+ continue;
+ }
+ const id = typeof artifact.id === 'string' && artifact.id ? artifact.id : null;
+ if (!id) errors.push('artifact.id must be a non-empty string');
+ else if (ids.has(id)) errors.push(`artifact id is duplicated: ${id}`);
+ else ids.add(id);
+ if (!isDigest(artifact.sha256)) errors.push(`artifact ${id || ''} sha256 is required`);
+ let relative;
+ try { relative = safeRelative(artifact.path, `artifact ${id || ''} path`); }
+ catch (error) { errors.push(error.message); continue; }
+ if (paths.has(relative)) {
+ errors.push(`artifact path is duplicated: ${relative}`);
+ continue;
+ }
+ paths.add(relative);
+ if (!root || !id || !isDigest(artifact.sha256)) continue;
+ const file = path.resolve(root, relative);
+ const fromRoot = path.relative(root, file);
+ if (fromRoot === '..' || fromRoot.startsWith(`..${path.sep}`) || path.isAbsolute(fromRoot)) {
+ errors.push(`artifact ${id} escapes evidence root`);
+ continue;
+ }
+ try {
+ let current = root;
+ for (const segment of relative.split('/')) {
+ current = path.join(current, segment);
+ const stat = fs.lstatSync(current);
+ if (stat.isSymbolicLink()) throw new Error('path must not contain symlinks');
+ }
+ const stat = fs.lstatSync(file);
+ if (!stat.isFile()) throw new Error('must be a regular file');
+ if (stat.nlink > 1) throw new Error('hardlinks are not allowed');
+ const canonicalFile = fs.realpathSync(file);
+ const canonicalRelative = path.relative(root, canonicalFile);
+ if (canonicalRelative === '..' || canonicalRelative.startsWith(`..${path.sep}`) || path.isAbsolute(canonicalRelative)) throw new Error('resolves outside evidence root');
+ const actual = sha256Bytes(fs.readFileSync(file));
+ if (actual !== artifact.sha256) throw new Error('digest mismatch');
+ verifiedIds.add(id);
+ } catch (error) {
+ errors.push(`artifact ${id} cannot be verified: ${error.message}`);
+ }
+ }
+ return verifiedIds;
+}
+
+module.exports = { stableValue, stableJson, sha256Bytes, contentId, isDigest, safeRelative, verifyArtifacts };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-experiment.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-experiment.js
new file mode 100644
index 0000000..a6f51fa
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-experiment.js
@@ -0,0 +1,234 @@
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { runTask, writeAtomic } = require('./sg-task-runner.js');
+const { readTaskManifest } = require('./agent-task-manifest.js');
+const { contentId, stableJson, sha256Bytes, safeRelative, isDigest } = require('./sg-evidence-utils.js');
+
+function wilson(successes, total, z = 1.959963984540054) {
+ if (!Number.isInteger(successes) || !Number.isInteger(total) || total < 0 || successes < 0 || successes > total) throw new Error('invalid Wilson interval counts');
+ if (total === 0) return { lower: null, upper: null };
+ const p = successes / total;
+ const denominator = 1 + z * z / total;
+ const center = (p + z * z / (2 * total)) / denominator;
+ const margin = z * Math.sqrt((p * (1 - p) + z * z / (4 * total)) / total) / denominator;
+ return { lower: Math.max(0, center - margin), upper: Math.min(1, center + margin) };
+}
+
+function validateExperimentSpec(spec) {
+ const errors = [];
+ if (!spec || typeof spec !== 'object' || Array.isArray(spec)) return { valid: false, errors: ['experiment spec must be an object'] };
+ const allowed = new Set(['experimentVersion', 'experimentId', 'title', 'agent', 'tasks', 'repetitions', 'seed']);
+ for (const key of Object.keys(spec)) if (!allowed.has(key)) errors.push(`unknown experiment field: ${key}`);
+ if (spec.experimentVersion !== '1.0') errors.push('experimentVersion must be 1.0');
+ if (typeof spec.title !== 'string' || !spec.title) errors.push('title is required');
+ if (!spec.agent || typeof spec.agent !== 'object' || Array.isArray(spec.agent)) errors.push('agent must be an object');
+ else {
+ for (const key of Object.keys(spec.agent)) if (!['kind', 'provider', 'model', 'argv', 'adapter', 'infraExitCodes'].includes(key)) errors.push(`unknown agent field: ${key}`);
+ if (!['scripted', 'ai'].includes(spec.agent.kind)) errors.push('agent.kind must be scripted or ai');
+ for (const field of ['provider', 'model']) if (typeof spec.agent[field] !== 'string' || !spec.agent[field]) errors.push(`agent.${field} is required`);
+ if (!Array.isArray(spec.agent.argv) || !spec.agent.argv.length || spec.agent.argv.some((item) => typeof item !== 'string' || item.includes('\0'))) errors.push('agent.argv must be a non-empty safe string array');
+ else if (!spec.agent.argv.some((item) => item.includes('{adapter}'))) errors.push('agent.argv must invoke the content-bound {adapter}');
+ if (!spec.agent.adapter || typeof spec.agent.adapter !== 'object' || Array.isArray(spec.agent.adapter)) errors.push('agent.adapter is required');
+ else {
+ for (const key of Object.keys(spec.agent.adapter)) if (!['path', 'sha256'].includes(key)) errors.push(`unknown agent.adapter field: ${key}`);
+ try { safeRelative(spec.agent.adapter.path, 'agent.adapter.path'); } catch (error) { errors.push(error.message); }
+ if (!isDigest(spec.agent.adapter.sha256)) errors.push('agent.adapter.sha256 must be sha256:');
+ }
+ if (spec.agent.infraExitCodes !== undefined && (!Array.isArray(spec.agent.infraExitCodes) || !spec.agent.infraExitCodes.length || spec.agent.infraExitCodes.some((item) => !Number.isInteger(item) || item < 1 || item > 255) || new Set(spec.agent.infraExitCodes).size !== spec.agent.infraExitCodes.length)) errors.push('agent.infraExitCodes must be a non-empty array of unique exit codes from 1 to 255');
+ }
+ if (!Array.isArray(spec.tasks) || !spec.tasks.length) errors.push('tasks must be a non-empty array');
+ const ids = new Set();
+ for (const [index, task] of (spec.tasks || []).entries()) {
+ if (!task || typeof task !== 'object' || Array.isArray(task)) { errors.push(`tasks[${index}] must be an object`); continue; }
+ for (const key of Object.keys(task)) if (!['id', 'manifest', 'taskId', 'manifestSha256'].includes(key)) errors.push(`tasks[${index}] unknown field: ${key}`);
+ if (typeof task.id !== 'string' || !/^[a-z][a-z0-9-]{1,63}$/.test(task.id)) errors.push(`tasks[${index}].id must be a stable slug`);
+ else if (ids.has(task.id)) errors.push(`duplicate task id: ${task.id}`); else ids.add(task.id);
+ try { safeRelative(task.manifest, `tasks[${index}].manifest`); } catch (error) { errors.push(error.message); }
+ if (!isDigest(task.taskId)) errors.push(`tasks[${index}].taskId must be sha256:`);
+ if (!isDigest(task.manifestSha256)) errors.push(`tasks[${index}].manifestSha256 must be sha256:`);
+ }
+ if (!Number.isInteger(spec.repetitions) || spec.repetitions < 1 || spec.repetitions > 100) errors.push('repetitions must be 1..100');
+ if (!['string', 'number'].includes(typeof spec.seed) || typeof spec.seed === 'number' && !Number.isSafeInteger(spec.seed)) errors.push('seed must be a string or safe integer');
+ const material = { ...spec }; delete material.experimentId;
+ const expectedId = contentId('experiment', material, []);
+ if (spec.experimentId !== expectedId) errors.push(`experimentId mismatch (expected ${expectedId})`);
+ return { valid: errors.length === 0, errors, expectedId };
+}
+
+function aggregate(trials, agentKind) {
+ const valid = trials.filter((trial) => !['input-error', 'infra-error'].includes(trial.verdict));
+ const passed = valid.filter((trial) => trial.verdict === 'passed').length;
+ const durations = valid.map((trial) => trial.durationMs).sort((a, b) => a - b);
+ const counts = Object.fromEntries([...new Set(trials.map((trial) => trial.verdict))].sort().map((key) => [key, trials.filter((trial) => trial.verdict === key).length]));
+ const stage = (predicate) => ({ passed: valid.filter(predicate).length, total: valid.length });
+ const evidenceStage = (field, requirementField) => {
+ const required = valid.filter((trial) => trial[requirementField] > 0);
+ return {
+ passed: required.filter((trial) => trial[field].length === trial[requirementField] && trial[field].every((value) => value === 'passed')).length,
+ total: required.length,
+ };
+ };
+ const providerMetadata = trials.map((trial) => trial.providerMetadata).filter(Boolean);
+ const actualModels = [...new Set(providerMetadata.flatMap((metadata) => Object.keys(metadata.modelUsage || {})))].sort();
+ const usage = providerMetadata.reduce((sum, metadata) => ({
+ inputTokens: sum.inputTokens + Number(metadata.usage && metadata.usage.input_tokens || 0),
+ outputTokens: sum.outputTokens + Number(metadata.usage && metadata.usage.output_tokens || 0),
+ cacheReadInputTokens: sum.cacheReadInputTokens + Number(metadata.usage && metadata.usage.cache_read_input_tokens || 0),
+ totalCostUsd: sum.totalCostUsd + Number(metadata.totalCostUsd || 0),
+ }), { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, totalCostUsd: 0 });
+ const runtimeStage = evidenceStage('runtimeStatuses', 'runtimeRequired');
+ const visualStage = evidenceStage('visualStatuses', 'visualRequired');
+ return {
+ totalTrials: trials.length,
+ validTrials: valid.length,
+ invalidTrials: trials.filter((trial) => trial.verdict === 'input-error').length,
+ infraErrors: trials.filter((trial) => trial.verdict === 'infra-error').length,
+ passed,
+ passRate: valid.length ? passed / valid.length : null,
+ confidence95: wilson(passed, valid.length),
+ verdicts: counts,
+ stages: {
+ agentCompleted: stage((trial) => trial.agentExitCode === 0),
+ patchApplied: stage((trial) => trial.patchStatus === 'applied'),
+ policyCompliant: stage((trial) => trial.patchStatus === 'applied' && trial.verdict !== 'policy-violation'),
+ graderPassed: stage((trial) => trial.gradeVerdicts.length > 0 && trial.gradeVerdicts.every((value) => value === 'passed')),
+ runtimePassed: runtimeStage,
+ visualPassed: visualStage,
+ },
+ provider: { actualModels, usage },
+ durationMs: {
+ mean: durations.length ? durations.reduce((sum, value) => sum + value, 0) / durations.length : null,
+ median: durations.length ? durations.length % 2 ? durations[Math.floor(durations.length / 2)] : (durations[durations.length / 2 - 1] + durations[durations.length / 2]) / 2 : null,
+ },
+ claimLevel: agentKind === 'scripted'
+ ? 'harness-only'
+ : passed === 0
+ ? 'ai-task-contract-not-demonstrated'
+ : runtimeStage.total > 0 && runtimeStage.passed === runtimeStage.total && visualStage.total > 0 && visualStage.passed === visualStage.total
+ ? 'ai-task-contract-with-runtime-visual-subset'
+ : 'ai-static-contract-only',
+ };
+}
+
+function renderExperiment(report) {
+ const p = report.summary.passRate === null ? 'N/A' : `${(report.summary.passRate * 100).toFixed(1)}%`;
+ return `# Agent success-rate experiment\n\n- Experiment: \`${report.experimentId}\`\n- Agent: ${report.agent.provider || 'unknown'} / ${report.agent.model || 'unknown'} (${report.agent.kind})\n- Trials: ${report.summary.totalTrials}; valid ${report.summary.validTrials}; infra ${report.summary.infraErrors}\n- Pass rate: **${p}** (Wilson 95% ${report.summary.confidence95.lower === null ? 'N/A' : `${(report.summary.confidence95.lower * 100).toFixed(1)}%–${(report.summary.confidence95.upper * 100).toFixed(1)}%`})\n- Claim level: **${report.summary.claimLevel}**\n- Actual model(s): ${report.summary.provider.actualModels.join(', ') || 'not reported'}\n- Tokens: input ${report.summary.provider.usage.inputTokens}; output ${report.summary.provider.usage.outputTokens}; cache read ${report.summary.provider.usage.cacheReadInputTokens}\n- Recorded provider cost: $${report.summary.provider.usage.totalCostUsd.toFixed(6)}\n\n## Failure taxonomy\n\n${Object.entries(report.summary.verdicts).map(([key, value]) => `- ${key}: ${value}`).join('\n') || '- none'}\n\n## Trials\n\n${report.trials.map((trial) => `- ${trial.trialId}: ${trial.verdict}; patch=${trial.patchStatus || 'none'}; grades=${trial.gradeVerdicts.join(',') || 'none'}`).join('\n')}\n`;
+}
+
+function runExperiment({ specFile, outputRoot } = {}) {
+ const absoluteSpec = path.resolve(specFile);
+ const specBytes = fs.readFileSync(absoluteSpec);
+ const specSha256 = sha256Bytes(specBytes);
+ const spec = JSON.parse(specBytes);
+ const validation = validateExperimentSpec(spec);
+ if (!validation.valid) throw new Error('invalid ExperimentSpec: ' + validation.errors.join('; '));
+ const base = path.dirname(absoluteSpec);
+ const adapterFile = path.resolve(base, safeRelative(spec.agent.adapter.path, 'agent.adapter.path'));
+ const adapterRelative = path.relative(base, adapterFile);
+ if (adapterRelative === '..' || adapterRelative.startsWith(`..${path.sep}`) || path.isAbsolute(adapterRelative)) throw new Error('agent adapter escapes experiment directory');
+ let adapterBytes;
+ try {
+ const stat = fs.lstatSync(adapterFile);
+ if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('must be a regular non-symlink file');
+ adapterBytes = fs.readFileSync(adapterFile);
+ } catch (error) { throw new Error('agent adapter cannot be verified: ' + error.message); }
+ const actualAdapterSha256 = sha256Bytes(adapterBytes);
+ if (actualAdapterSha256 !== spec.agent.adapter.sha256) throw new Error(`agent adapter digest mismatch: expected ${spec.agent.adapter.sha256}, received ${actualAdapterSha256}`);
+ const root = path.resolve(outputRoot);
+ if (fs.existsSync(root) && fs.readdirSync(root).length) throw new Error('experiment output root must be empty');
+ fs.mkdirSync(root, { recursive: true });
+ const agentArgv = spec.agent.argv.map((argument) => argument.split('{experimentDir}').join(base).split('{adapter}').join(adapterFile).split('{node}').join(process.execPath));
+ const taskDefinitions = spec.tasks.map((task) => {
+ const manifestFile = path.resolve(base, task.manifest);
+ try {
+ const manifestBytes = fs.readFileSync(manifestFile);
+ const manifest = readTaskManifest(manifestFile);
+ const inputErrors = [];
+ const manifestSha256 = sha256Bytes(manifestBytes);
+ if (task.taskId !== manifest.taskId) inputErrors.push(`taskId mismatch: expected ${task.taskId}, received ${manifest.taskId}`);
+ if (task.manifestSha256 !== manifestSha256) inputErrors.push(`manifest digest mismatch: expected ${task.manifestSha256}, received ${manifestSha256}`);
+ return {
+ ...task,
+ manifestFile,
+ runtimeRequired: manifest.evidence.runtime.length,
+ visualRequired: manifest.evidence.visual.length,
+ inputError: inputErrors.length ? inputErrors.join('; ') : null,
+ };
+ } catch (error) {
+ return { ...task, manifestFile, runtimeRequired: 0, visualRequired: 0, inputError: error.message };
+ }
+ });
+ const trials = [];
+ for (const task of taskDefinitions) {
+ for (let repetition = 1; repetition <= spec.repetitions; repetition += 1) {
+ const trialId = `${task.id}-r${String(repetition).padStart(2, '0')}`;
+ const trialRoot = path.join(root, 'trials', trialId);
+ const started = process.hrtime.bigint();
+ try {
+ if (task.inputError) {
+ const error = new Error(task.inputError);
+ error.kind = 'input-error';
+ throw error;
+ }
+ const currentSpecSha256 = sha256Bytes(fs.readFileSync(absoluteSpec));
+ const currentAdapterSha256 = sha256Bytes(fs.readFileSync(adapterFile));
+ const currentManifestBytes = fs.readFileSync(task.manifestFile);
+ let currentManifest;
+ try { currentManifest = JSON.parse(currentManifestBytes); }
+ catch (parseError) { currentManifest = null; }
+ const perTrialInputErrors = [];
+ if (currentSpecSha256 !== specSha256) perTrialInputErrors.push(`experiment spec digest mismatch: expected ${specSha256}, received ${currentSpecSha256}`);
+ if (currentAdapterSha256 !== spec.agent.adapter.sha256) perTrialInputErrors.push(`agent adapter digest mismatch: expected ${spec.agent.adapter.sha256}, received ${currentAdapterSha256}`);
+ const currentManifestSha256 = sha256Bytes(currentManifestBytes);
+ if (currentManifestSha256 !== task.manifestSha256) perTrialInputErrors.push(`task manifest digest mismatch: expected ${task.manifestSha256}, received ${currentManifestSha256}`);
+ if (!currentManifest || currentManifest.taskId !== task.taskId) perTrialInputErrors.push(`taskId mismatch: expected ${task.taskId}, received ${currentManifest && currentManifest.taskId}`);
+ if (perTrialInputErrors.length) {
+ const error = new Error(perTrialInputErrors.join('; '));
+ error.kind = 'input-error';
+ throw error;
+ }
+ const result = runTask({
+ taskFile: task.manifestFile,
+ agentArgv,
+ artifactRoot: trialRoot,
+ agent: { provider: spec.agent.provider, model: spec.agent.model, infraExitCodes: spec.agent.infraExitCodes || [] },
+ trialId,
+ integrityInputs: [
+ { id: 'experiment-spec', file: absoluteSpec, sha256: specSha256 },
+ { id: 'agent-adapter', file: adapterFile, sha256: spec.agent.adapter.sha256 },
+ ],
+ });
+ const report = result.report;
+ const results = report.grades.flatMap((grade) => grade.results);
+ trials.push({
+ trialId, taskSlug: task.id, taskId: task.taskId, runId: report.runId, verdict: report.verdict,
+ agentExitCode: report.agent.exitCode, patchStatus: report.patch && report.patch.status,
+ gradeVerdicts: report.grades.map((grade) => grade.verdict),
+ runtimeRequired: task.runtimeRequired,
+ visualRequired: task.visualRequired,
+ runtimeStatuses: results.filter((item) => item.type === 'runtime-evidence').map((item) => item.status),
+ visualStatuses: results.filter((item) => item.type === 'visual-evidence').map((item) => item.status),
+ providerMetadata: report.agent.providerMetadata || null,
+ durationMs: Number(process.hrtime.bigint() - started) / 1e6,
+ artifactPath: path.relative(root, result.runFile).split(path.sep).join('/'),
+ });
+ } catch (error) {
+ trials.push({ trialId, taskSlug: task.id, taskId: task.taskId, runId: null, verdict: error.kind === 'input-error' ? 'input-error' : 'infra-error', agentExitCode: null, patchStatus: null, gradeVerdicts: [], runtimeRequired: task.runtimeRequired, visualRequired: task.visualRequired, runtimeStatuses: [], visualStatuses: [], providerMetadata: null, durationMs: Number(process.hrtime.bigint() - started) / 1e6, error: error.message, artifactPath: null });
+ }
+ }
+ }
+ const report = {
+ reportVersion: '1.0', resultId: null, experimentId: spec.experimentId,
+ specSha256, seed: spec.seed,
+ agent: { kind: spec.agent.kind, provider: spec.agent.provider, model: spec.agent.model },
+ taskSetSha256: sha256Bytes(stableJson(spec.tasks)), repetitions: spec.repetitions,
+ summary: aggregate(trials, spec.agent.kind), trials,
+ };
+ report.resultId = contentId('experiment-result', report, ['resultId']);
+ writeAtomic(path.join(root, 'experiment.json'), Buffer.from(JSON.stringify(report, null, 2) + '\n'));
+ writeAtomic(path.join(root, 'EXPERIMENT.md'), Buffer.from(renderExperiment(report)));
+ return report;
+}
+
+module.exports = { wilson, validateExperimentSpec, aggregate, renderExperiment, runExperiment };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-extraction-subject.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-extraction-subject.js
new file mode 100644
index 0000000..b12204d
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-extraction-subject.js
@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+'use strict';
+const { parentPort } = require('node:worker_threads');
+const { runExtraction } = require('./sg-pack-extract-core.js');
+
+parentPort.once('message', (message) => {
+ const port = message && message.port;
+ const token = message && message.token;
+ if (!port || typeof token !== 'string' || !message.configPath || !message.workspaceRoot) process.exit(1);
+ const send = port.postMessage.bind(port);
+ const close = port.close.bind(port);
+ parentPort.close();
+ try {
+ const extraction = runExtraction(message.configPath, { expectedLibDir: message.workspaceRoot });
+ send({
+ token,
+ result: {
+ ok: true,
+ validation: extraction.validation,
+ equivalence: extraction.equivalence,
+ consumedFiles: extraction.consumedFiles,
+ inventory: extraction.inventory,
+ },
+ });
+ close();
+ } catch (error) {
+ send({ token, result: { ok: false, kind: error.kind || 'gate', message: error.message } });
+ close();
+ }
+});
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-extraction-worker.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-extraction-worker.js
new file mode 100644
index 0000000..3351197
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-extraction-worker.js
@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+'use strict';
+const crypto = require('node:crypto');
+const path = require('node:path');
+const { MessageChannel, Worker } = require('node:worker_threads');
+
+const [, , configPath, workspaceRoot] = process.argv;
+if (!configPath || !workspaceRoot) {
+ process.stderr.write('extraction supervisor requires config path and workspace root\n');
+ process.exit(1);
+}
+
+const token = crypto.randomBytes(32).toString('hex');
+const channel = new MessageChannel();
+const worker = new Worker(path.join(__dirname, 'sg-extraction-subject.js'), { stdout: true, stderr: true });
+let message = null;
+let protocolError = null;
+let subjectStdoutBytes = 0;
+let subjectStderrBytes = 0;
+worker.stdout.on('data', (chunk) => { subjectStdoutBytes += chunk.length; });
+worker.stderr.on('data', (chunk) => { subjectStderrBytes += chunk.length; });
+channel.port1.on('message', (value) => {
+ if (message !== null) {
+ protocolError = 'subject sent more than one result';
+ return;
+ }
+ if (!value || value.token !== token || !value.result || typeof value.result !== 'object') {
+ protocolError = 'subject returned an invalid authenticated result';
+ return;
+ }
+ message = value.result;
+});
+worker.once('error', (error) => { protocolError = 'subject worker error: ' + error.message; });
+worker.once('exit', (exitCode) => {
+ channel.port1.close();
+ const protocolOk = exitCode === 0 && protocolError === null && message !== null;
+ process.stdout.write(JSON.stringify({
+ supervisorVersion: '1.0',
+ protocolOk,
+ workerExitCode: exitCode,
+ protocolError: protocolOk ? null : protocolError || 'subject exited without an authenticated trusted-wrapper result',
+ subjectStdoutBytes,
+ subjectStderrBytes,
+ result: protocolOk ? message : null,
+ }) + '\n');
+ process.exit(protocolOk ? 0 : 1);
+});
+worker.postMessage({ token, configPath, workspaceRoot, port: channel.port2 }, [channel.port2]);
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-grader.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-grader.js
new file mode 100644
index 0000000..3b1bfe8
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-grader.js
@@ -0,0 +1,264 @@
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { runCommand } = require('./sg-command-runner.js');
+const { executeRules } = require('./sg-pack-rules-core.js');
+const { readRuntimeEvidence } = require('./sg-runtime-evidence.js');
+const { readVisualEvidence } = require('./sg-visual-evidence.js');
+const { contentId, sha256Bytes, safeRelative } = require('./sg-evidence-utils.js');
+require('./sg-data-loader.js');
+
+const CHECK_TYPES = new Set(['data-pack-contract', 'library-rules', 'extract-equivalence', 'command', 'runtime-evidence', 'visual-evidence']);
+const CHECK_FIELDS = new Set(['id', 'type', 'weight', 'required', 'path', 'scenarioId', 'rulesPath', 'dataPath', 'configPath', 'argv', 'cwd', 'expectedExit', 'infraExitCodes', 'timeoutMs', 'maxOutputBytes', 'strict', 'ruleIds']);
+
+function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }
+function readJson(file) { return JSON.parse(fs.readFileSync(file, 'utf8')); }
+function resolveInRoot(root, value, label) {
+ const relative = safeRelative(value, label);
+ const file = path.resolve(root, relative);
+ const fromRoot = path.relative(root, file);
+ if (fromRoot === '..' || fromRoot.startsWith(`..${path.sep}`) || path.isAbsolute(fromRoot)) throw new Error(`${label} escapes workspace`);
+ return file;
+}
+
+function validateGraderSpec(spec) {
+ const errors = [];
+ if (!isObject(spec)) return { valid: false, errors: ['grader spec must be an object'] };
+ for (const field of Object.keys(spec)) if (!['graderVersion', 'graderId', 'title', 'checks'].includes(field)) errors.push(`unknown grader field: ${field}`);
+ if (spec.graderVersion !== '1.0') errors.push('graderVersion must be 1.0');
+ if (typeof spec.graderId !== 'string' || !/^[a-z][a-z0-9-]{1,63}$/.test(spec.graderId)) errors.push('graderId must be a stable slug');
+ if (!Array.isArray(spec.checks) || !spec.checks.length) errors.push('checks must be a non-empty array');
+ const ids = new Set();
+ for (const [index, check] of (spec.checks || []).entries()) {
+ if (!isObject(check)) { errors.push(`checks[${index}] must be an object`); continue; }
+ for (const field of Object.keys(check)) if (!CHECK_FIELDS.has(field)) errors.push(`checks[${index}] unknown field: ${field}`);
+ if (typeof check.id !== 'string' || !/^[a-z][a-z0-9-]{1,63}$/.test(check.id)) errors.push(`checks[${index}].id must be a stable slug`);
+ else if (ids.has(check.id)) errors.push(`duplicate check id: ${check.id}`); else ids.add(check.id);
+ if (!CHECK_TYPES.has(check.type)) errors.push(`checks[${index}].type is unsupported`);
+ if (typeof check.weight !== 'number' || !Number.isFinite(check.weight) || check.weight <= 0) errors.push(`checks[${index}].weight must be greater than 0`);
+ if (typeof check.required !== 'boolean') errors.push(`checks[${index}].required must be boolean`);
+ if (check.type === 'command' && (!Array.isArray(check.argv) || !check.argv.length || check.argv.some((item) => typeof item !== 'string'))) errors.push(`checks[${index}].argv must be a non-empty string array`);
+ if (check.infraExitCodes !== undefined && (check.type !== 'command' || !Array.isArray(check.infraExitCodes) || !check.infraExitCodes.length || check.infraExitCodes.some((item) => !Number.isInteger(item) || item < 1 || item > 255) || new Set(check.infraExitCodes).size !== check.infraExitCodes.length)) errors.push(`checks[${index}].infraExitCodes must be a non-empty array of unique exit codes from 1 to 255 for a command check`);
+ if (['runtime-evidence', 'visual-evidence'].includes(check.type) && (typeof check.scenarioId !== 'string' || !check.scenarioId)) errors.push(`checks[${index}].scenarioId is required for evidence checks`);
+ if (check.scenarioId !== undefined && !['runtime-evidence', 'visual-evidence'].includes(check.type)) errors.push(`checks[${index}].scenarioId is only valid for evidence checks`);
+ const requiredPaths = check.type === 'library-rules' ? ['rulesPath', 'dataPath'] : check.type === 'extract-equivalence' ? ['configPath'] : ['data-pack-contract', 'runtime-evidence', 'visual-evidence'].includes(check.type) ? ['path'] : [];
+ for (const field of requiredPaths) {
+ try { safeRelative(check[field], `checks[${index}].${field}`); } catch (error) { errors.push(error.message); }
+ }
+ if (check.cwd !== undefined) { try { safeRelative(check.cwd, `checks[${index}].cwd`); } catch (error) { errors.push(error.message); } }
+ }
+ return { valid: errors.length === 0, errors };
+}
+
+function baseResult(check) {
+ return { id: check.id, type: check.type, required: check.required, weight: check.weight, status: 'infra-error', score: 0, findings: [], evidence: {} };
+}
+
+function gradePackContract(check, root) {
+ const result = baseResult(check);
+ try {
+ const pack = readJson(resolveInRoot(root, check.path, `${check.id}.path`));
+ const validation = globalThis.SGDataLoader.validate(pack);
+ const failed = validation.errors.length > 0 || (check.strict && validation.warnings.length > 0);
+ result.status = failed ? 'failed' : 'passed';
+ result.score = failed ? 0 : check.weight;
+ result.evidence = { errors: validation.errors, warnings: validation.warnings, strict: Boolean(check.strict) };
+ } catch (error) {
+ result.status = 'invalid';
+ result.findings.push(error.message);
+ }
+ return result;
+}
+
+function gradeRules(check, root) {
+ const result = baseResult(check);
+ try {
+ const rules = readJson(resolveInRoot(root, check.rulesPath, `${check.id}.rulesPath`));
+ const pack = readJson(resolveInRoot(root, check.dataPath, `${check.id}.dataPath`));
+ const execution = executeRules(rules, pack, { ruleIds: check.ruleIds });
+ if (execution.structuralErrors.length || (execution.selectionErrors || []).length || (execution.unknownRuleIds || []).length) result.status = 'invalid';
+ else if (!execution.results.length || execution.results.every((item) => item.status === 'no-check')) result.status = 'not-assessed';
+ else {
+ const failed = execution.counts.hardFail > 0 || execution.counts.errors > 0 || (check.strict && execution.counts.softFail > 0);
+ result.status = failed ? 'failed' : 'passed';
+ }
+ result.score = result.status === 'passed' ? check.weight : 0;
+ result.evidence = execution;
+ } catch (error) {
+ result.status = 'invalid';
+ result.findings.push(error.message);
+ }
+ return result;
+}
+
+function gradeExtraction(check, root, options) {
+ const result = baseResult(check);
+ try {
+ const configPath = resolveInRoot(root, check.configPath, `${check.id}.configPath`);
+ const run = runCommand({
+ argv: [process.execPath, path.join(options.toolRoot, 'scripts', 'lib', 'sg-extraction-worker.js'), configPath, root],
+ cwd: root,
+ allowedRoot: root,
+ timeoutMs: check.timeoutMs || options.timeoutMs || 120000,
+ maxOutputBytes: check.maxOutputBytes || options.maxOutputBytes || 1048576,
+ env: options.env || {},
+ envAllowlist: options.envAllowlist,
+ });
+ let supervisor = null;
+ try { supervisor = JSON.parse(run.stdout.trim()); }
+ catch (_) { supervisor = null; }
+ const workerResult = supervisor && supervisor.protocolOk === true && supervisor.result && typeof supervisor.result === 'object'
+ ? supervisor.result
+ : null;
+ result.evidence = {
+ exitCode: run.exitCode, signal: run.signal, timedOut: run.timedOut, durationMs: run.durationMs,
+ stdoutSha256: sha256Bytes(run.stdout), stderrSha256: sha256Bytes(run.stderr),
+ stdout: run.stdout, stderr: run.stderr, stdoutTruncated: run.stdoutTruncated, stderrTruncated: run.stderrTruncated,
+ error: run.error || null, supervisor, worker: workerResult,
+ };
+ if (run.error && !run.timedOut && run.exitCode === null) result.status = 'infra-error';
+ else if (run.timedOut || !workerResult || workerResult.ok !== true) {
+ result.status = workerResult && workerResult.kind === 'input' ? 'invalid' : 'failed';
+ if (workerResult && workerResult.message) result.findings.push(workerResult.message);
+ else result.findings.push(run.timedOut ? 'candidate extraction worker timed out' : 'candidate extraction worker did not return a valid result');
+ } else {
+ const failed = workerResult.validation.errors.length > 0 || workerResult.equivalence.diffs.length > 0 || !workerResult.equivalence.coverage.complete;
+ result.status = failed ? 'failed' : 'passed';
+ result.evidence.validation = workerResult.validation;
+ result.evidence.equivalence = workerResult.equivalence;
+ result.evidence.consumedFiles = workerResult.consumedFiles;
+ }
+ result.score = result.status === 'passed' ? check.weight : 0;
+ } catch (error) {
+ result.status = 'invalid';
+ result.findings.push(error.message);
+ }
+ return result;
+}
+
+function commandArgv(argv, root, options) {
+ const replacements = {
+ '{node}': process.execPath,
+ '{workspace}': root,
+ '{artifacts}': options.artifactRoot || '',
+ '{treeSha256}': options.treeSha256 || '',
+ '{taskId}': options.taskId || '',
+ '{grader}': options.graderRoot || '',
+ '{toolRoot}': options.toolRoot || path.resolve(__dirname, '..', '..'),
+ };
+ return argv.map((argument) => Object.entries(replacements).reduce((value, [token, replacement]) => value.split(token).join(replacement), argument));
+}
+
+function gradeCommand(check, root, options) {
+ const result = baseResult(check);
+ try {
+ const cwd = check.cwd ? resolveInRoot(root, check.cwd, `${check.id}.cwd`) : root;
+ const run = runCommand({
+ argv: commandArgv(check.argv, root, options),
+ cwd,
+ allowedRoot: root,
+ timeoutMs: check.timeoutMs || options.timeoutMs || 120000,
+ maxOutputBytes: check.maxOutputBytes || options.maxOutputBytes || 1048576,
+ env: options.env || {},
+ envAllowlist: options.envAllowlist,
+ });
+ const expectedExit = check.expectedExit === undefined ? 0 : check.expectedExit;
+ const infraExitCodes = new Set(check.infraExitCodes || []);
+ if (run.error && !run.timedOut && run.exitCode === null || infraExitCodes.has(run.exitCode)) result.status = 'infra-error';
+ else result.status = run.exitCode === expectedExit && !run.timedOut ? 'passed' : 'failed';
+ result.score = result.status === 'passed' ? check.weight : 0;
+ result.evidence = {
+ argv: run.argv, cwd: path.relative(root, run.cwd) || '.', exitCode: run.exitCode, signal: run.signal,
+ timedOut: run.timedOut, durationMs: run.durationMs,
+ stdoutSha256: sha256Bytes(run.stdout), stderrSha256: sha256Bytes(run.stderr),
+ stdout: run.stdout, stderr: run.stderr,
+ stdoutTruncated: run.stdoutTruncated, stderrTruncated: run.stderrTruncated, error: run.error || null,
+ };
+ } catch (error) {
+ result.status = 'invalid';
+ result.findings.push(error.message);
+ }
+ return result;
+}
+
+function gradeEvidence(check, root, kind, options) {
+ const result = baseResult(check);
+ try {
+ const evidenceRoot = options.artifactRoot ? fs.realpathSync(path.resolve(options.artifactRoot)) : root;
+ const file = resolveInRoot(evidenceRoot, check.path, `${check.id}.path`);
+ const loaded = kind === 'runtime' ? readRuntimeEvidence(file) : readVisualEvidence(file);
+ const expectedTree = options.treeSha256;
+ const boundTree = kind === 'runtime' ? loaded.evidence.subjectTreeSha256 : loaded.evidence.candidateTreeSha256;
+ if (expectedTree && boundTree !== expectedTree) {
+ loaded.validation.errors.push(`evidence tree digest mismatch: expected ${expectedTree}, received ${boundTree}`);
+ loaded.validation.valid = false;
+ loaded.validation.status = 'not-assessed';
+ }
+ const scenarioMatched = kind === 'runtime'
+ ? loaded.evidence.scenarioId === check.scenarioId
+ : Array.isArray(loaded.evidence.scenarios) && loaded.evidence.scenarios.some((scenario) => scenario && scenario.id === check.scenarioId);
+ if (!scenarioMatched) {
+ loaded.validation.errors.push(`evidence does not contain required scenario: ${check.scenarioId}`);
+ loaded.validation.valid = false;
+ loaded.validation.status = 'not-assessed';
+ }
+ result.status = loaded.validation.valid ? loaded.validation.status : 'failed';
+ result.score = result.status === 'passed' ? check.weight : 0;
+ result.evidence = { scenarioId: check.scenarioId, file: path.relative(evidenceRoot, file), evidenceId: loaded.evidence.evidenceId, validation: loaded.validation };
+ } catch (error) {
+ result.status = error.code === 'ENOENT' ? 'not-assessed' : 'failed';
+ result.findings.push(error.message);
+ }
+ return result;
+}
+
+function gradeWorkspace({ spec, workspaceRoot, graderRoot = null, artifactRoot = null, toolRoot = path.resolve(__dirname, '..', '..'), taskId = null, treeSha256 = null, timeoutMs, maxOutputBytes, env, envAllowlist } = {}) {
+ const validation = validateGraderSpec(spec);
+ if (!validation.valid) {
+ const error = new Error('Invalid GraderSpec: ' + validation.errors.join('; '));
+ error.code = 'INVALID_GRADER_SPEC';
+ error.errors = validation.errors;
+ throw error;
+ }
+ const root = fs.realpathSync(path.resolve(workspaceRoot));
+ const options = { timeoutMs, maxOutputBytes, env, envAllowlist, graderRoot, artifactRoot, toolRoot, taskId, treeSha256 };
+ const results = spec.checks.map((check) => {
+ if (check.type === 'data-pack-contract') return gradePackContract(check, root);
+ if (check.type === 'library-rules') return gradeRules(check, root);
+ if (check.type === 'extract-equivalence') return gradeExtraction(check, root, options);
+ if (check.type === 'command') return gradeCommand(check, root, options);
+ if (check.type === 'runtime-evidence') return gradeEvidence(check, root, 'runtime', options);
+ return gradeEvidence(check, root, 'visual', options);
+ });
+ const requiredIncomplete = results.some((item) => item.required && item.status !== 'passed');
+ const infraError = results.some((item) => item.status === 'infra-error');
+ const failed = results.some((item) => ['failed', 'invalid'].includes(item.status));
+ const requiredNotAssessed = results.some((item) => item.required && item.status === 'not-assessed');
+ const candidateFailure = failed || requiredNotAssessed;
+ const score = results.reduce((sum, item) => sum + item.score, 0);
+ const maximumScore = results.reduce((sum, item) => sum + item.weight, 0);
+ const report = {
+ gradeVersion: '1.0',
+ gradeId: null,
+ graderId: spec.graderId,
+ taskId,
+ treeSha256,
+ verdict: candidateFailure ? 'failed' : infraError ? 'infra-error' : requiredIncomplete ? 'failed' : 'passed',
+ score,
+ maximumScore,
+ results,
+ summary: {
+ total: results.length,
+ passed: results.filter((item) => item.status === 'passed').length,
+ failed: results.filter((item) => item.status === 'failed').length,
+ notAssessed: results.filter((item) => item.status === 'not-assessed').length,
+ invalid: results.filter((item) => item.status === 'invalid').length,
+ infraErrors: results.filter((item) => item.status === 'infra-error').length,
+ requiredIncomplete: results.filter((item) => item.required && item.status !== 'passed').length,
+ },
+ };
+ report.gradeId = contentId('grade', report, ['gradeId']);
+ return report;
+}
+
+module.exports = { validateGraderSpec, gradeWorkspace };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-grader.schema.json b/research/agent-eval/results/executed-tooling/scripts/lib/sg-grader.schema.json
new file mode 100644
index 0000000..bffa1b0
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-grader.schema.json
@@ -0,0 +1,41 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://sg.local/contracts/grader-spec-v1.json",
+ "title": "SG Task Grader Spec v1.0",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["graderVersion", "graderId", "checks"],
+ "properties": {
+ "graderVersion": { "const": "1.0" },
+ "graderId": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,63}$" },
+ "title": { "type": "string", "minLength": 1 },
+ "checks": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["id", "type", "weight", "required"],
+ "properties": {
+ "id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,63}$" },
+ "type": { "enum": ["data-pack-contract", "library-rules", "extract-equivalence", "command", "runtime-evidence", "visual-evidence"] },
+ "weight": { "type": "number", "exclusiveMinimum": 0 },
+ "required": { "type": "boolean" },
+ "path": { "type": "string", "minLength": 1 },
+ "scenarioId": { "type": "string", "minLength": 1 },
+ "rulesPath": { "type": "string", "minLength": 1 },
+ "dataPath": { "type": "string", "minLength": 1 },
+ "configPath": { "type": "string", "minLength": 1 },
+ "argv": { "type": "array", "minItems": 1, "items": { "type": "string" } },
+ "cwd": { "type": "string", "minLength": 1 },
+ "expectedExit": { "type": "integer", "minimum": 0, "maximum": 255 },
+ "infraExitCodes": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "integer", "minimum": 1, "maximum": 255 } },
+ "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 600000 },
+ "maxOutputBytes": { "type": "integer", "minimum": 1024, "maximum": 16777216 },
+ "strict": { "type": "boolean" },
+ "ruleIds": { "type": "array", "items": { "type": "string", "minLength": 1 } }
+ }
+ }
+ }
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-candidate.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-candidate.js
new file mode 100644
index 0000000..140d5e0
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-candidate.js
@@ -0,0 +1,428 @@
+'use strict';
+
+/*
+ * Pure Review Decisions -> Candidate Pack builder.
+ * File I/O, output paths, and process exit codes belong to sg-pack-candidate.js.
+ */
+const { sha256, buildCandidateReview, verifyCandidateReview } = require('./sg-recrawl-review.js');
+const { buildDiff } = require('./sg-pack-diff.js');
+require('./sg-data-loader.js');
+
+const loader = globalThis.SGDataLoader;
+const SUPPORTED_ACTIONS = new Set(['apply', 'keep', 'map-alias', 'reject']);
+const RESERVED_FIELDS = new Set(['kind', '__proto__', 'prototype', 'constructor']);
+const RESERVED_KEYS = new Set(['__proto__', 'prototype', 'constructor']);
+
+class CandidateError extends Error {
+ constructor(message, exitCode = 2) {
+ super(message);
+ this.name = 'CandidateError';
+ this.exitCode = exitCode;
+ }
+}
+
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function deepClone(value) {
+ return JSON.parse(JSON.stringify(value));
+}
+
+function hasOwn(object, key) {
+ return Object.prototype.hasOwnProperty.call(object, key);
+}
+
+function sameValue(a, b) {
+ return JSON.stringify(a) === JSON.stringify(b);
+}
+
+function requireString(value, label) {
+ if (typeof value !== 'string' || !value.trim()) throw new CandidateError(`${label} must be a non-empty string`);
+ return value;
+}
+
+function validateConfidence(value, label) {
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 1) {
+ throw new CandidateError(`${label} must be a number between 0 and 1`);
+ }
+}
+
+function pathText(section, id, field) {
+ return `${section}.${id}.${field}`;
+}
+
+function validateReportShape(report) {
+ if (!isObject(report)) throw new CandidateError('review report must be an object');
+ const review = report.candidateReview;
+ if (!isObject(review)) throw new CandidateError('review report is missing candidateReview');
+ if (!Array.isArray(review.observations) || !Array.isArray(review.reviewItems)) {
+ throw new CandidateError('candidateReview observations and reviewItems must be arrays');
+ }
+ if (new Set(review.reviewItems.map((item) => item && item.itemId)).size !== review.reviewItems.length) {
+ throw new CandidateError('candidate review contains duplicate review item ids');
+ }
+ const allItemMap = new Map();
+ const reviewItemMap = new Map();
+ const observationMap = new Map();
+ for (const observation of review.observations) {
+ if (!isObject(observation) || typeof observation.recordId !== 'string') throw new CandidateError('candidate observation has an invalid recordId');
+ if (observationMap.has(observation.recordId)) throw new CandidateError(`duplicate candidate observation: ${observation.recordId}`);
+ observationMap.set(observation.recordId, observation);
+ if (observation.resolution && observation.resolution.status === 'hit') {
+ if (!Array.isArray(observation.checks)) throw new CandidateError(`${observation.recordId}.checks must be an array`);
+ for (const check of observation.checks) {
+ if (!isObject(check) || typeof check.itemId !== 'string') throw new CandidateError(`${observation.recordId} has an invalid field check`);
+ if (allItemMap.has(check.itemId)) throw new CandidateError(`duplicate candidate check: ${check.itemId}`);
+ allItemMap.set(check.itemId, { type: 'field', observation, check });
+ }
+ }
+ if (observation.resolution && observation.resolution.status === 'miss') {
+ const identityId = `${observation.recordId}:identity`;
+ if (allItemMap.has(identityId)) throw new CandidateError(`duplicate candidate identity: ${identityId}`);
+ allItemMap.set(identityId, { type: 'identity', observation });
+ }
+ }
+ for (const item of review.reviewItems) {
+ if (!isObject(item) || typeof item.itemId !== 'string') throw new CandidateError('candidate review item has an invalid itemId');
+ if (!allItemMap.has(item.itemId)) throw new CandidateError(`review item does not resolve to an observation: ${item.itemId}`);
+ const source = allItemMap.get(item.itemId);
+ if (source.type === 'field' && source.check.classification !== item.classification) {
+ throw new CandidateError(`review item classification mismatch: ${item.itemId}`);
+ }
+ if (source.type === 'identity' && item.classification !== 'miss') {
+ throw new CandidateError(`identity review item must be classified as miss: ${item.itemId}`);
+ }
+ reviewItemMap.set(item.itemId, source);
+ }
+ for (const [itemId, source] of allItemMap) {
+ const requiresReview = source.type === 'identity' ||
+ (source.type === 'field' && ['gap', 'conflict', 'unsupported'].includes(source.check.classification));
+ if (requiresReview && !reviewItemMap.has(itemId)) {
+ throw new CandidateError(`candidate review omitted required item: ${itemId}`);
+ }
+ }
+ return { review, itemMap: reviewItemMap, allItemMap, observationMap };
+}
+
+function validateDecisions(decisions, reviewId, itemMap) {
+ if (!isObject(decisions) || decisions.decisionsVersion !== '1.0') {
+ throw new CandidateError('review decisions must use decisionsVersion "1.0"');
+ }
+ if (decisions.reportId !== reviewId) throw new CandidateError('review decisions reportId does not match candidate review report');
+ requireString(decisions.reviewedBy, 'reviewedBy');
+ requireString(decisions.reviewedAt, 'reviewedAt');
+ if (!Array.isArray(decisions.decisions)) throw new CandidateError('review decisions must contain a decisions array');
+ const seen = new Set();
+ for (const decision of decisions.decisions) {
+ if (!isObject(decision)) throw new CandidateError('each review decision must be an object');
+ requireString(decision.itemId, 'decision.itemId');
+ if (seen.has(decision.itemId)) throw new CandidateError(`duplicate decision for review item: ${decision.itemId}`);
+ seen.add(decision.itemId);
+ if (!itemMap.has(decision.itemId)) throw new CandidateError(`decision references unknown review item: ${decision.itemId}`);
+ requireString(decision.action, `decision ${decision.itemId}.action`);
+ if (!SUPPORTED_ACTIONS.has(decision.action)) throw new CandidateError(`unsupported decision action: ${decision.action}`);
+ }
+ const expected = [...itemMap.keys()];
+ const missing = expected.filter((itemId) => !seen.has(itemId));
+ const extra = [...seen].filter((itemId) => !itemMap.has(itemId));
+ if (missing.length || extra.length) {
+ throw new CandidateError(
+ `review decisions are incomplete (missing: ${missing.join(', ') || 'none'}; extra: ${extra.join(', ') || 'none'})`,
+ 1,
+ );
+ }
+ return decisions;
+}
+
+function checkCurrentBaseline(entity, check, entityId) {
+ const exists = hasOwn(entity, check.baselineField);
+ const current = exists ? entity[check.baselineField] : undefined;
+ if (exists !== Boolean(check.baselineExists) || !sameValue(current, check.baseline)) {
+ throw new CandidateError(`stale entity field at ${pathText('entities', entityId, check.baselineField)}`, 1);
+ }
+}
+
+function displayField(pack, entity) {
+ const fields = pack.kindNameFields;
+ return isObject(fields) && hasOwn(fields, entity.kind) && fields[entity.kind] ? fields[entity.kind] : 'name';
+}
+
+function validateAliasTarget(pack, entityId) {
+ if (typeof entityId !== 'string' || !hasOwn(pack.entities || {}, entityId)) {
+ throw new CandidateError(`map-alias target must be an existing entity: ${entityId}`);
+ }
+}
+
+function aliasValue(target, context) {
+ if (context === undefined) return target;
+ if (typeof context !== 'string' || !context.trim()) throw new CandidateError('alias context must be a non-empty string');
+ return { id: target, context };
+}
+
+function applyAlias(pack, observation, decision, audit) {
+ const alias = observation.crawledName;
+ const target = decision.entityId;
+ validateAliasTarget(pack, target);
+ if (RESERVED_KEYS.has(alias)) throw new CandidateError(`reserved alias key is not allowed: ${alias}`);
+ if (hasOwn(pack.entities, alias)) throw new CandidateError(`alias collides with entity id: ${alias}`);
+ if (hasOwn(pack.aliases || {}, alias)) {
+ const current = pack.aliases[alias];
+ const currentId = isObject(current) ? current.id : current;
+ if (currentId !== target) throw new CandidateError(`existing alias points to a different entity: ${alias}`);
+ const requested = aliasValue(target, decision.context);
+ if (decision.context !== undefined && !sameValue(current, requested)) {
+ throw new CandidateError(`existing alias has different reviewed context: ${alias}`);
+ }
+ audit.result = 'noop-already-applied';
+ audit.before = current;
+ audit.after = current;
+ return;
+ }
+ if (!isObject(pack.aliases)) pack.aliases = {};
+ const next = aliasValue(target, decision.context);
+ pack.aliases[alias] = next;
+ audit.result = 'applied';
+ audit.before = null;
+ audit.after = next;
+}
+
+function mergeFieldOrigin(entityProvenance, check, decision, context) {
+ if (!isObject(entityProvenance)) entityProvenance = {};
+ if (!isObject(entityProvenance.fieldOrigins)) entityProvenance.fieldOrigins = {};
+ const field = check.baselineField;
+ const origin = {
+ origin: context.review.origin,
+ sourceUrl: context.review.sourceUrl,
+ fetchedAt: context.review.fetchedAt,
+ confidence: decision.confidence,
+ note: decision.note || `Applied reviewed crawl field ${check.crawledField}`,
+ reportId: context.review.reportId,
+ recordId: context.observation.recordId,
+ itemId: check.itemId,
+ reviewedBy: context.decisions.reviewedBy,
+ reviewedAt: context.decisions.reviewedAt,
+ };
+ entityProvenance.fieldOrigins[field] = origin;
+ return entityProvenance;
+}
+
+function applyField(pack, observation, check, decision, context, audit) {
+ const entityId = observation.resolution.entityId;
+ const entity = pack.entities[entityId];
+ if (!entity) throw new CandidateError(`field decision references missing entity: ${entityId}`);
+ if (decision.action === 'keep') {
+ audit.result = 'kept-baseline';
+ audit.before = check.baseline;
+ audit.after = check.baseline;
+ return;
+ }
+ if (check.classification === 'unsupported') {
+ throw new CandidateError(`unsupported review item may only use keep: ${check.itemId}`);
+ }
+ if (!['gap', 'conflict'].includes(check.classification)) {
+ throw new CandidateError(`apply is not valid for ${check.classification}: ${check.itemId}`);
+ }
+ validateConfidence(decision.confidence, `decision ${decision.itemId}.confidence`);
+ const field = check.baselineField;
+ if (RESERVED_FIELDS.has(field) || field === displayField(pack, entity)) {
+ throw new CandidateError(`cannot update reserved or display entity field: ${field}`);
+ }
+ if (field.includes('.') || field.includes('[') || field.includes(']')) {
+ throw new CandidateError(`nested entity field updates are not supported: ${field}`);
+ }
+ if (typeof check.crawled !== 'string' || !check.crawled) {
+ throw new CandidateError(`applied crawl value must be a non-empty string: ${check.itemId}`);
+ }
+ const before = hasOwn(entity, field) ? entity[field] : undefined;
+ if (before === check.crawled) {
+ audit.result = 'noop-already-applied';
+ audit.before = before;
+ audit.after = before;
+ return;
+ }
+ entity[field] = check.crawled;
+ audit.result = 'applied';
+ audit.before = before;
+ audit.after = check.crawled;
+ if (!isObject(pack.provenance)) pack.provenance = {};
+ if (!isObject(pack.provenance.entities)) pack.provenance.entities = {};
+ let entityProvenance = pack.provenance.entities[entityId];
+ if (!entityProvenance) {
+ entityProvenance = {
+ origin: 'baseline-untracked',
+ sourceUrl: null,
+ fetchedAt: context.review.fetchedAt,
+ confidence: 1,
+ note: 'Baseline record had no provenance; reviewed crawl evidence is recorded per field in fieldOrigins.',
+ };
+ }
+ pack.provenance.entities[entityId] = mergeFieldOrigin(entityProvenance, check, decision, context);
+ audit.provenance = pack.provenance.entities[entityId].fieldOrigins[field];
+}
+
+function validateFieldOperations(baselinePack, itemMap, decisions) {
+ const writes = new Map();
+ const decisionsByItem = new Map(decisions.decisions.map((decision) => [decision.itemId, decision]));
+ for (const [itemId, source] of itemMap) {
+ if (source.type !== 'field') continue;
+ const entityId = source.observation.resolution.entityId;
+ const entity = baselinePack.entities && baselinePack.entities[entityId];
+ if (!entity) throw new CandidateError(`field review item references missing entity: ${entityId}`, 1);
+ checkCurrentBaseline(entity, source.check, entityId);
+ const decision = decisionsByItem.get(itemId);
+ if (!decision || decision.action !== 'apply') continue;
+ const key = `${entityId}\u0000${source.check.baselineField}`;
+ const value = source.check.crawled;
+ const current = writes.get(key);
+ if (current && current.value !== value) {
+ throw new CandidateError(`conflicting reviewed values for entities.${entityId}.${source.check.baselineField}`);
+ }
+ writes.set(key, { value, itemId });
+ }
+}
+
+function validateAllowedSections(before, after) {
+ const allowed = new Set(['aliases', 'entities', 'provenance']);
+ const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
+ for (const key of keys) {
+ if (allowed.has(key)) continue;
+ if (!sameValue(before[key], after[key])) throw new CandidateError(`candidate changed unsupported section: ${key}`, 1);
+ }
+ const beforeAliases = before.aliases || {};
+ const afterAliases = after.aliases || {};
+ for (const key of Object.keys(beforeAliases)) {
+ if (!sameValue(beforeAliases[key], afterAliases[key])) throw new CandidateError(`candidate rewrote existing alias: ${key}`, 1);
+ }
+ const beforeEntities = before.entities || {};
+ const afterEntities = after.entities || {};
+ for (const id of Object.keys(beforeEntities)) {
+ if (!hasOwn(afterEntities, id)) throw new CandidateError(`candidate removed entity: ${id}`, 1);
+ const fields = new Set([...Object.keys(beforeEntities[id]), ...Object.keys(afterEntities[id])]);
+ for (const field of fields) {
+ if (field === 'kind' || field === 'name') {
+ if (!sameValue(beforeEntities[id][field], afterEntities[id][field])) throw new CandidateError(`candidate changed protected entity field: entities.${id}.${field}`, 1);
+ }
+ }
+ }
+ for (const id of Object.keys(afterEntities)) {
+ if (!hasOwn(beforeEntities, id)) throw new CandidateError(`candidate added entity: ${id}`, 1);
+ }
+}
+
+function buildCandidate({ baselinePack, baselineBytes, report, decisions, decisionsBytes, strict = false, recordsBytes, rawRecords }) {
+ if (!isObject(baselinePack)) throw new CandidateError('baseline Data Pack must be an object', 2);
+ const baselineValidation = loader.validate(baselinePack);
+ if (baselineValidation.errors.length) throw new CandidateError(`baseline Data Pack is invalid: ${baselineValidation.errors.join('; ')}`, 1);
+ let candidateReview;
+ try {
+ candidateReview = verifyCandidateReview(report, baselineBytes, recordsBytes);
+ } catch (error) {
+ throw new CandidateError(error.message, 2);
+ }
+ let rebuiltReview;
+ try {
+ rebuiltReview = buildCandidateReview({
+ pack: baselinePack,
+ rawRecords,
+ baselineBytes,
+ recordsBytes,
+ origin: candidateReview.origin,
+ sourceUrl: candidateReview.sourceUrl,
+ fetchedAt: candidateReview.fetchedAt,
+ });
+ } catch (error) {
+ throw new CandidateError(`candidate review cannot be reproduced: ${error.message}`, 2);
+ }
+ if (!sameValue(rebuiltReview, candidateReview)) {
+ throw new CandidateError('candidate review does not match a fresh cross-check of baseline and records', 2);
+ }
+ const reportContext = validateReportShape(report);
+ validateDecisions(decisions, candidateReview.reportId, reportContext.itemMap);
+ validateFieldOperations(baselinePack, reportContext.itemMap, decisions);
+
+ const candidate = deepClone(baselinePack);
+ const auditOperations = [];
+ const context = { review: candidateReview, decisions };
+ const decisionsByItem = new Map(decisions.decisions.map((decision) => [decision.itemId, decision]));
+ for (const itemId of reportContext.itemMap.keys()) {
+ const decision = decisionsByItem.get(itemId);
+ const source = reportContext.itemMap.get(decision.itemId);
+ const audit = {
+ decisionId: decision.itemId,
+ itemId: decision.itemId,
+ action: decision.action,
+ target: null,
+ result: null,
+ before: null,
+ after: null,
+ note: decision.note || null,
+ };
+ if (source.type === 'identity') {
+ audit.target = `aliases.${source.observation.crawledName}`;
+ if (decision.action === 'map-alias') {
+ if (source.observation.rawFields && Object.keys(source.observation.rawFields).length) {
+ // Identity approval must not silently turn un-cross-checked fields into entity writes.
+ audit.note = audit.note || 'Identity mapped; business fields require a follow-up recrawl review.';
+ }
+ applyAlias(candidate, source.observation, decision, audit);
+ } else if (decision.action === 'reject') {
+ audit.result = 'rejected';
+ } else {
+ throw new CandidateError(`identity review item requires map-alias or reject: ${decision.itemId}`);
+ }
+ } else {
+ const check = source.check;
+ audit.target = pathText('entities', source.observation.resolution.entityId, check.baselineField);
+ if (decision.action === 'map-alias' || decision.action === 'reject') {
+ throw new CandidateError(`field review item cannot use ${decision.action}: ${decision.itemId}`);
+ }
+ applyField(candidate, source.observation, check, decision, { ...context, observation: source.observation }, audit);
+ }
+ auditOperations.push(audit);
+ }
+
+ validateAllowedSections(baselinePack, candidate);
+ const validation = loader.validate(candidate);
+ if (validation.errors.length) throw new CandidateError(`candidate Data Pack is invalid: ${validation.errors.join('; ')}`, 1);
+ if (strict && validation.warnings.length) throw new CandidateError(`strict candidate validation failed: ${validation.warnings.join('; ')}`, 1);
+
+ const candidateBytes = Buffer.from(JSON.stringify(candidate, null, 2) + '\n', 'utf8');
+ const diff = buildDiff(baselinePack, candidate, { old: 'baseline', new: 'candidate' });
+ const audit = {
+ auditVersion: '1.0',
+ status: 'valid',
+ inputs: {
+ baselineSha256: sha256(baselineBytes),
+ recordsSha256: sha256(recordsBytes),
+ reportId: candidateReview.reportId,
+ decisionsSha256: sha256(decisionsBytes),
+ candidateSha256: sha256(candidateBytes),
+ },
+ review: {
+ reviewedBy: decisions.reviewedBy,
+ reviewedAt: decisions.reviewedAt,
+ },
+ operations: auditOperations,
+ unresolved: [],
+ validation: {
+ errors: validation.errors,
+ warnings: validation.warnings,
+ },
+ diff: {
+ entities: diff.entities,
+ aliases: diff.aliases,
+ provenance: diff.provenance,
+ total: diff.total,
+ },
+ derivationsImpacted: diff.derivationsImpacted,
+ };
+ return { candidate, candidateBytes, audit, auditBytes: Buffer.from(JSON.stringify(audit, null, 2) + '\n', 'utf8') };
+}
+
+module.exports = {
+ CandidateError,
+ buildCandidate,
+ validateReportShape,
+ validateDecisions,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-diff.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-diff.js
new file mode 100644
index 0000000..0ac8c9a
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-diff.js
@@ -0,0 +1,249 @@
+'use strict';
+
+/* Pure structural diff and derivation-impact helpers. */
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function changedPaths(a, b, prefix = '') {
+ const out = [];
+ if (JSON.stringify(a) === JSON.stringify(b)) return out;
+ if (!isObject(a) || !isObject(b)) {
+ out.push(prefix || '(value)');
+ return out;
+ }
+ const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
+ for (const key of keys) {
+ const current = prefix ? `${prefix}.${key}` : key;
+ if (!Object.prototype.hasOwnProperty.call(a, key)) out.push(`${current} (added)`);
+ else if (!Object.prototype.hasOwnProperty.call(b, key)) out.push(`${current} (removed)`);
+ else if (JSON.stringify(a[key]) !== JSON.stringify(b[key])) {
+ if (isObject(a[key]) && isObject(b[key])) out.push(...changedPaths(a[key], b[key], current));
+ else out.push(current);
+ }
+ }
+ return out;
+}
+
+function diffRecordMap(oldMap, newMap) {
+ const oldValue = oldMap || {};
+ const newValue = newMap || {};
+ const oldKeys = Object.keys(oldValue);
+ const newKeys = Object.keys(newValue);
+ const has = (object, key) => Object.prototype.hasOwnProperty.call(object, key);
+ const added = newKeys.filter((key) => !has(oldValue, key));
+ const removed = oldKeys.filter((key) => !has(newValue, key));
+ const changed = [];
+ for (const key of newKeys) {
+ if (!Object.prototype.hasOwnProperty.call(oldValue, key)) continue;
+ const fields = changedPaths(oldValue[key], newValue[key]);
+ if (fields.length) changed.push({ id: key, fields });
+ }
+ return { added, removed, changed };
+}
+
+function relationKey(relation) {
+ if (relation && typeof relation.id === 'string' && relation.id) return `id:${relation.id}`;
+ const scope = Array.isArray(relation && relation.scope) ? [...new Set(relation.scope)].sort().join('|') : '*';
+ return `${relation && relation.a}::${relation && relation.b}::${relation && relation.type}::scope=${scope}`;
+}
+
+function recordMapWithOccurrences(records, keyOf) {
+ const out = new Map();
+ const seen = new Map();
+ for (const record of records || []) {
+ const base = keyOf(record);
+ const occurrence = (seen.get(base) || 0) + 1;
+ seen.set(base, occurrence);
+ out.set(occurrence === 1 ? base : `${base}#${occurrence}`, record);
+ }
+ return out;
+}
+
+function diffRelations(oldRelations, newRelations) {
+ const oldMap = recordMapWithOccurrences(oldRelations, relationKey);
+ const newMap = recordMapWithOccurrences(newRelations, relationKey);
+ const added = [...newMap.keys()].filter((key) => !oldMap.has(key));
+ const removed = [...oldMap.keys()].filter((key) => !newMap.has(key));
+ const changed = [];
+ for (const key of newMap.keys()) {
+ if (!oldMap.has(key)) continue;
+ const fields = changedPaths(oldMap.get(key), newMap.get(key));
+ if (fields.length) changed.push({ id: key, fields });
+ }
+ return { added, removed, changed };
+}
+
+function stageMap(stages) {
+ const out = Object.create(null);
+ const seen = Object.create(null);
+ (stages || []).forEach((stage, index) => {
+ const base = stage && (stage.key || stage.id) || `#${index}`;
+ seen[base] = (seen[base] || 0) + 1;
+ const key = seen[base] === 1 ? base : `${base}#${seen[base]}`;
+ out[key] = stage;
+ });
+ return out;
+}
+
+function stageOrder(stages) {
+ return (stages || []).map((stage, index) => stage && (stage.key || stage.id) || `#${index}`);
+}
+
+function stageOrderChanged(oldStages, newStages) {
+ const oldOrder = stageOrder(oldStages);
+ const newOrder = stageOrder(newStages);
+ if (oldOrder.length !== newOrder.length) return false;
+ if (JSON.stringify([...oldOrder].sort()) !== JSON.stringify([...newOrder].sort())) return false;
+ return oldOrder.some((key, index) => key !== newOrder[index]);
+}
+
+function diffPairs(oldPairs, newPairs) {
+ const normalize = (pairs) => new Set((pairs || []).map((pair) => JSON.stringify([...pair].sort())));
+ const oldSet = normalize(oldPairs);
+ const newSet = normalize(newPairs);
+ return {
+ added: [...newSet].filter((value) => !oldSet.has(value)),
+ removed: [...oldSet].filter((value) => !newSet.has(value)),
+ };
+}
+
+function count(section) {
+ return (section.added ? section.added.length : 0) +
+ (section.removed ? section.removed.length : 0) +
+ (section.changed ? section.changed.length : 0);
+}
+
+function matchSegments(sourceSegments, changedSegments) {
+ if (changedSegments.length < sourceSegments.length) {
+ for (let index = 0; index < changedSegments.length; index += 1) {
+ if (sourceSegments[index] !== '*' && sourceSegments[index] !== changedSegments[index]) return false;
+ }
+ return true;
+ }
+ for (let index = 0; index < sourceSegments.length; index += 1) {
+ if (sourceSegments[index] !== '*' && sourceSegments[index] !== changedSegments[index]) return false;
+ }
+ return true;
+}
+
+function sourceMatches(sourcePath, section, id, field) {
+ const segments = String(sourcePath || '').split('.');
+ if (segments[0] !== section) return false;
+ if (segments.length === 1) return true;
+ const changed = [];
+ if (id !== undefined && id !== null) changed.push(String(id));
+ if (field) changed.push(...String(field).split('.'));
+ return matchSegments(segments.slice(1), changed);
+}
+
+function impactedDerivations(pack, section, id, field) {
+ const hits = [];
+ for (const [name, derivation] of Object.entries((pack && pack.derivations) || {})) {
+ const sources = [derivation.source].concat(derivation.alsoTouches || []);
+ if (sources.some((source) => sourceMatches(source, section, id, field))) hits.push({ name, ...derivation });
+ }
+ return hits;
+}
+
+function collectImpacts(pack, report, oldPack, newPack) {
+ const impacts = new Map();
+ const add = (derivation, trigger) => {
+ if (!impacts.has(derivation.name)) {
+ impacts.set(derivation.name, {
+ name: derivation.name,
+ kind: derivation.kind,
+ note: derivation.note,
+ consumers: derivation.consumers,
+ triggers: new Set(),
+ });
+ }
+ impacts.get(derivation.name).triggers.add(trigger);
+ };
+ const feedRecordSection = (section, label) => {
+ for (const id of report[section].added) {
+ for (const derivation of impactedDerivations(pack, section, id)) add(derivation, `+${label} ${id}`);
+ }
+ for (const id of report[section].removed) {
+ for (const derivation of impactedDerivations(pack, section, id)) add(derivation, `-${label} ${id}`);
+ }
+ for (const change of report[section].changed) {
+ const fields = change.fields.map((field) => field.replace(/ \(added\)| \(removed\)/g, ''));
+ for (const field of [...new Set(fields)]) {
+ for (const derivation of impactedDerivations(pack, section, change.id, field)) {
+ add(derivation, `~${label} ${change.id}.${field}`);
+ }
+ }
+ }
+ };
+
+ feedRecordSection('entities', 'entity');
+ feedRecordSection('aliases', 'alias');
+ if (count(report.relations)) for (const derivation of impactedDerivations(pack, 'relations')) add(derivation, 'relations changed');
+ feedRecordSection('contents', 'content');
+ if (count(report.stages) || report.stages.orderChanged) {
+ for (const derivation of impactedDerivations(pack, 'stages')) add(derivation, report.stages.orderChanged ? 'stages order changed' : 'stages changed');
+ }
+
+ const oldDomain = oldPack.domain || {};
+ const newDomain = newPack.domain || {};
+ if (JSON.stringify(oldDomain) !== JSON.stringify(newDomain)) {
+ const paths = changedPaths(oldDomain, newDomain).map((field) => field.replace(/ \(added\)| \(removed\)/g, ''));
+ for (const field of paths) {
+ for (const derivation of impactedDerivations(pack, 'domain', undefined, field)) add(derivation, `~domain.${field}`);
+ }
+ }
+
+ for (const change of report.derivations.changed) {
+ add({ name: change.id, kind: 'meta', note: 'derivation definition changed', consumers: [] }, `~derivation ${change.id}`);
+ }
+ for (const section of ['assets', 'provenance', 'attributeTypes', 'relationTypes', 'heroRelTypes', 'kindNameFields', 'meta']) {
+ if (count(report[section])) {
+ for (const derivation of impactedDerivations(pack, section)) add(derivation, `${section} changed`);
+ }
+ }
+ return [...impacts.values()].map((impact) => ({ ...impact, triggers: [...impact.triggers] }));
+}
+
+function buildDiff(oldPack, newPack, labels = {}) {
+ const report = {
+ old: labels.old || 'old',
+ new: labels.new || 'new',
+ entities: diffRecordMap(oldPack.entities, newPack.entities),
+ relations: diffRelations(oldPack.relations, newPack.relations),
+ aliases: diffRecordMap(oldPack.aliases, newPack.aliases),
+ contents: diffRecordMap(oldPack.contents, newPack.contents),
+ stages: diffRecordMap(stageMap(oldPack.stages), stageMap(newPack.stages)),
+ sameAs: diffPairs(oldPack.sameAs, newPack.sameAs),
+ derivations: diffRecordMap(oldPack.derivations, newPack.derivations),
+ assets: diffRecordMap(oldPack.assets, newPack.assets),
+ provenance: diffRecordMap(oldPack.provenance, newPack.provenance),
+ attributeTypes: diffRecordMap(oldPack.attributeTypes, newPack.attributeTypes),
+ relationTypes: diffRecordMap(oldPack.relationTypes, newPack.relationTypes),
+ heroRelTypes: diffRecordMap(oldPack.heroRelTypes, newPack.heroRelTypes),
+ kindNameFields: diffRecordMap(oldPack.kindNameFields, newPack.kindNameFields),
+ domain: diffRecordMap(oldPack.domain, newPack.domain),
+ meta: diffRecordMap(isObject(oldPack.meta) ? oldPack.meta : {}, isObject(newPack.meta) ? newPack.meta : {}),
+ };
+ const orderChanged = stageOrderChanged(oldPack.stages, newPack.stages);
+ if (orderChanged) report.stages.orderChanged = true;
+ report.derivationsImpacted = collectImpacts(newPack, report, oldPack, newPack);
+ report.total = count(report.entities) + count(report.relations) + count(report.aliases) +
+ count(report.contents) + count(report.stages) + (orderChanged ? 1 : 0) + report.sameAs.added.length + report.sameAs.removed.length +
+ count(report.derivations) + count(report.assets) + count(report.provenance) +
+ count(report.attributeTypes) + count(report.relationTypes) + count(report.heroRelTypes) +
+ count(report.kindNameFields) + count(report.domain) + count(report.meta);
+ return report;
+}
+
+module.exports = {
+ isObject,
+ changedPaths,
+ diffRecordMap,
+ diffRelations,
+ diffPairs,
+ count,
+ sourceMatches,
+ impactedDerivations,
+ buildDiff,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-extract-core.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-extract-core.js
new file mode 100644
index 0000000..7286dbd
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-extract-core.js
@@ -0,0 +1,334 @@
+'use strict';
+
+/* Read-only extraction/equivalence collector used by product reports. */
+class ExtractionError extends Error {
+ constructor(message, kind = 'gate') {
+ super(message);
+ this.name = 'ExtractionError';
+ this.kind = kind;
+ }
+}
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const acorn = require('../vendor/acorn.js');
+require('./sg-data-loader.js');
+
+function sha1File(file) {
+ return 'sha1:' + crypto.createHash('sha1').update(fs.readFileSync(file)).digest('hex');
+}
+
+function sha256Bytes(bytes) {
+ return 'sha256:' + crypto.createHash('sha256').update(bytes).digest('hex');
+}
+
+function sha256File(file) {
+ return sha256Bytes(fs.readFileSync(file));
+}
+
+function sliceLiteral(source, spec, libDir) {
+ const match = spec.pattern.exec(source);
+ if (!match) throw new Error(`Failed to locate literal "${spec.key}": pattern did not match`);
+ const start = match.index + match[0].length;
+ if (spec.json) {
+ const end = source.indexOf('', start);
+ if (end === -1) throw new Error(`Literal "${spec.key}": closing tag not found`);
+ return JSON.parse(source.slice(start, end).trim());
+ }
+ const node = acorn.parseExpressionAt(source, start, { ecmaVersion: 'latest' });
+ const literalSource = source.slice(node.start, node.end);
+ const context = spec.ctx || {};
+ const fn = new Function(...Object.keys(context), 'return (' + literalSource + ');');
+ return fn(...Object.values(context));
+}
+
+function sliceLiterals(engineSource, specs, libDir, consumedFiles) {
+ const out = {};
+ const cache = { __engine__: engineSource };
+ for (const spec of specs || []) {
+ let source = engineSource;
+ if (spec.file) {
+ if (!cache[spec.file]) {
+ const literalFile = path.resolve(libDir, spec.file);
+ const literalBytes = fs.readFileSync(literalFile);
+ if (consumedFiles) consumedFiles[literalFile] = sha256Bytes(literalBytes);
+ cache[spec.file] = literalBytes.toString('utf8');
+ }
+ source = cache[spec.file];
+ }
+ out[spec.key] = sliceLiteral(source, spec, libDir);
+ }
+ return out;
+}
+
+function deepEqual(left, right, pathName = '$', diffs = []) {
+ if (left === right) return diffs;
+ if (typeof left !== typeof right) {
+ diffs.push(`${pathName}: type mismatch ${typeof left} vs ${typeof right}`);
+ return diffs;
+ }
+ if (left === null || right === null || typeof left !== 'object') {
+ if (left !== right) diffs.push(`${pathName}: ${JSON.stringify(left)} !== ${JSON.stringify(right)}`);
+ return diffs;
+ }
+ if (Array.isArray(left) !== Array.isArray(right)) {
+ diffs.push(`${pathName}: array/object mismatch`);
+ return diffs;
+ }
+ if (Array.isArray(left)) {
+ if (left.length !== right.length) diffs.push(`${pathName}: array length ${left.length} vs ${right.length}`);
+ const length = Math.max(left.length, right.length);
+ for (let i = 0; i < length; i += 1) {
+ if (i < left.length && i < right.length) deepEqual(left[i], right[i], `${pathName}[${i}]`, diffs);
+ }
+ return diffs;
+ }
+ const leftKeys = Object.keys(left);
+ const rightKeys = Object.keys(right);
+ for (const key of leftKeys) if (!Object.prototype.hasOwnProperty.call(right, key)) diffs.push(`${pathName}.${key}: missing on right`);
+ for (const key of rightKeys) if (!Object.prototype.hasOwnProperty.call(left, key)) diffs.push(`${pathName}.${key}: missing on left`);
+ for (const key of leftKeys) if (Object.prototype.hasOwnProperty.call(right, key)) deepEqual(left[key], right[key], `${pathName}.${key}`, diffs);
+ return diffs;
+}
+
+function resolveAssetRoot(libDir, assetDir = 'lib/assets') {
+ if (typeof assetDir !== 'string' || !assetDir || /[\u0000-\u001f\u007f]/.test(assetDir)) throw new Error(`Unsafe assetDir: ${assetDir}`);
+ if (path.isAbsolute(assetDir) || /^[A-Za-z]:[\\/]/.test(assetDir)) throw new Error('Extraction config assetDir must be relative to libDir');
+ const libraryRoot = path.resolve(libDir);
+ const root = path.resolve(libraryRoot, assetDir);
+ const relative = path.relative(libraryRoot, root);
+ if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) throw new Error(`Extraction config assetDir escapes libDir: ${assetDir}`);
+ return root;
+}
+
+function assetRelativePath(assetPath, assetBase = '') {
+ const normalized = assetPath.replace(/\\/g, '/');
+ const normalizedBase = typeof assetBase === 'string' ? assetBase.replace(/\\/g, '/') : '';
+ if (normalizedBase && normalized.startsWith(normalizedBase)) return normalized.slice(normalizedBase.length);
+ return normalized.replace(/^(\.\.\/)?assets\//, '');
+}
+
+function localAssetPath(libDir, assetPath, assetDir = 'lib/assets', assetBase = '') {
+ if (typeof assetPath !== 'string' || !assetPath || /[\u0000-\u001f\u007f]/.test(assetPath)) throw new Error(`Unsafe asset path: ${assetPath}`);
+ const root = resolveAssetRoot(libDir, assetDir);
+ const relative = assetRelativePath(assetPath, assetBase);
+ if (!relative || path.isAbsolute(relative) || /^[A-Za-z]:[\\/]/.test(relative)) throw new Error(`Asset path must be relative: ${assetPath}`);
+ const absolute = path.resolve(root, relative);
+ const fromRoot = path.relative(root, absolute);
+ if (fromRoot === '..' || fromRoot.startsWith(`..${path.sep}`) || path.isAbsolute(fromRoot)) throw new Error(`Asset path escapes assetDir ${assetDir}: ${assetPath}`);
+ if (!fs.existsSync(root)) return absolute;
+ if (fs.lstatSync(root).isSymbolicLink()) throw new Error('Asset root must not be a symlink');
+ let current = root;
+ for (const segment of relative.split(/[\\/]/)) {
+ current = path.join(current, segment);
+ if (fs.existsSync(current) && fs.lstatSync(current).isSymbolicLink()) throw new Error(`Asset path contains a symlink: ${assetPath}`);
+ }
+ return absolute;
+}
+
+function collectAssets(pack, libDir, consumedFiles, assetDir = 'lib/assets') {
+ const found = Object.create(null);
+ const assetBase = pack.meta && typeof pack.meta.assetBase === 'string' ? pack.meta.assetBase : '';
+ const walk = (value) => {
+ if (typeof value === 'string') {
+ if (/^(https?:|data:)/i.test(value)) return;
+ const match = /(\.\.\/)?assets\/[^\s"'`)]*/.exec(value);
+ if (match && /\.(png|jpe?g|webp|gif|svg)$/i.test(match[0])) found[match[0]] = true;
+ else if (assetBase && /\.(png|jpe?g|webp|gif|svg)$/i.test(value)) found[assetBase + value] = true;
+ } else if (Array.isArray(value)) value.forEach(walk);
+ else if (value && typeof value === 'object') Object.values(value).forEach(walk);
+ };
+ walk(pack);
+ for (const assetPath of Object.keys(pack.assets || {})) {
+ if (!/^(https?:|data:)/i.test(assetPath)) found[assetPath] = true;
+ }
+ const assets = {};
+ for (const assetPath of Object.keys(found)) {
+ const absolute = localAssetPath(libDir, assetPath, assetDir, assetBase);
+ try {
+ const stat = fs.statSync(absolute);
+ if (consumedFiles) consumedFiles[path.resolve(absolute)] = sha256File(absolute);
+ assets[assetPath] = { exists: true, bytes: stat.size, hash: sha1File(absolute) };
+ } catch (_) {
+ assets[assetPath] = { exists: false };
+ }
+ }
+ return assets;
+}
+
+function equivalenceCoverage(config) {
+ const extracted = (config.literals || []).map((item) => item.key);
+ const mapped = (config.equivalence || []).map((item) => item.lit);
+ const ignored = (config.equivalenceIgnore || []).map((item) => item.lit);
+ return {
+ extracted,
+ mapped,
+ ignored,
+ unmapped: extracted.filter((key) => !mapped.includes(key) && !ignored.includes(key)),
+ complete: extracted.every((key) => mapped.includes(key) || ignored.includes(key)),
+ };
+}
+
+function runEquivalence(config, defaults, pack) {
+ global.window = globalThis;
+ const enginePath = path.resolve(config.libDir, config.engineFile);
+ delete require.cache[require.resolve(enginePath)];
+ require(enginePath);
+ const library = globalThis[config.globalName];
+ if (!library || typeof library.__fromPack !== 'function') throw new Error(`Engine does not export __fromPack (${config.globalName}); complete the engine patch first`);
+ const restored = library.__fromPack(pack);
+ if (!restored || typeof restored !== 'object' || Array.isArray(restored)) throw new Error('__fromPack(pack) must return an object for equivalence checks');
+ const diffs = [];
+ for (const mapping of config.equivalence || []) {
+ if (!Object.prototype.hasOwnProperty.call(defaults, mapping.lit)) throw new Error(`Equivalence literal does not exist: ${mapping.lit}`);
+ if (!Object.prototype.hasOwnProperty.call(restored, mapping.from)) throw new Error(`__fromPack(pack) did not return equivalence field: ${mapping.from}`);
+ let left = defaults[mapping.lit];
+ let right = restored[mapping.from];
+ const sorter = config.sortKeys && config.sortKeys[mapping.lit];
+ if (sorter) {
+ if (!Array.isArray(left) || !Array.isArray(right)) throw new Error(`sortKeys.${mapping.lit} requires arrays on both sides`);
+ left = left.slice().sort(sorter);
+ right = right.slice().sort(sorter);
+ }
+ deepEqual(left, right, `$.${mapping.lit}`, diffs);
+ }
+ return { comparisons: (config.equivalence || []).length, diffs, coverage: equivalenceCoverage(config) };
+}
+
+function validateRelativeConfigPath(value, field) {
+ if (typeof value !== 'string' || !value.trim()) throw new Error(`Extraction config ${field} must be a non-empty string`);
+ if (/[\u0000-\u001f\u007f]/.test(value) || path.isAbsolute(value) || /^[A-Za-z]:[\\/]/.test(value)) throw new Error(`Extraction config ${field} must be a safe path relative to libDir`);
+ const normalized = value.replace(/\\/g, '/');
+ if (normalized.split('/').includes('..')) throw new Error(`Extraction config ${field} must not escape libDir`);
+}
+
+function validateExtractionConfig(config) {
+ if (!config || typeof config !== 'object') throw new Error('Extraction config must export an object');
+ for (const field of ['libId', 'libDir', 'engineFile', 'globalName']) {
+ if (typeof config[field] !== 'string' || !config[field].trim()) throw new Error(`Extraction config ${field} must be a non-empty string`);
+ }
+ validateRelativeConfigPath(config.engineFile, 'engineFile');
+ if (config.assetDir !== undefined) validateRelativeConfigPath(config.assetDir, 'assetDir');
+ if (!Array.isArray(config.literals) || !config.literals.length) throw new Error('Extraction config literals must be a non-empty array');
+ const literalKeys = new Set();
+ for (const [index, literal] of config.literals.entries()) {
+ if (!literal || typeof literal !== 'object') throw new Error(`Extraction config literals[${index}] must be an object`);
+ if (typeof literal.key !== 'string' || !literal.key.trim()) throw new Error(`Extraction config literals[${index}].key must be a non-empty string`);
+ if (literalKeys.has(literal.key)) throw new Error(`Extraction config literal key is duplicated: ${literal.key}`);
+ literalKeys.add(literal.key);
+ if (!(literal.pattern instanceof RegExp)) throw new Error(`Extraction config literal ${literal.key} pattern must be a RegExp`);
+ if (literal.file !== undefined) validateRelativeConfigPath(literal.file, `literal ${literal.key} file`);
+ }
+ if (typeof config.buildPack !== 'function') throw new Error('Extraction config buildPack must be a function');
+ if (!Array.isArray(config.equivalence)) throw new Error('Extraction config equivalence must be an array');
+ if (config.equivalenceIgnore !== undefined && !Array.isArray(config.equivalenceIgnore)) throw new Error('Extraction config equivalenceIgnore must be an array');
+ const accounted = new Set();
+ for (const [index, mapping] of config.equivalence.entries()) {
+ if (!mapping || typeof mapping !== 'object') throw new Error(`Extraction config equivalence[${index}] must be an object`);
+ for (const field of ['lit', 'from']) if (typeof mapping[field] !== 'string' || !mapping[field].trim()) throw new Error(`Extraction config equivalence[${index}].${field} must be a non-empty string`);
+ if (!literalKeys.has(mapping.lit)) throw new Error(`Extraction config equivalence references unknown literal: ${mapping.lit}`);
+ if (accounted.has(mapping.lit)) throw new Error(`Extraction config literal has duplicate equivalence coverage: ${mapping.lit}`);
+ accounted.add(mapping.lit);
+ }
+ for (const [index, ignored] of (config.equivalenceIgnore || []).entries()) {
+ if (!ignored || typeof ignored !== 'object') throw new Error(`Extraction config equivalenceIgnore[${index}] must be an object`);
+ if (typeof ignored.lit !== 'string' || !ignored.lit.trim()) throw new Error(`Extraction config equivalenceIgnore[${index}].lit must be a non-empty string`);
+ if (typeof ignored.reason !== 'string' || !ignored.reason.trim()) throw new Error(`Extraction config equivalenceIgnore[${index}].reason must be a non-empty string`);
+ if (!literalKeys.has(ignored.lit)) throw new Error(`Extraction config equivalenceIgnore references unknown literal: ${ignored.lit}`);
+ if (accounted.has(ignored.lit)) throw new Error(`Extraction config literal has duplicate equivalence coverage: ${ignored.lit}`);
+ accounted.add(ignored.lit);
+ }
+ const missing = [...literalKeys].filter((key) => !accounted.has(key));
+ if (missing.length) throw new Error(`Extraction config literals lack equivalence coverage: ${missing.join(', ')}; map them or use equivalenceIgnore with a reason`);
+ return config;
+}
+
+function runExtractionUnsafe(configPath, options = {}) {
+ const modulesBefore = new Set(Object.keys(require.cache));
+ const absoluteConfig = path.resolve(configPath);
+ const consumedFiles = {};
+ consumedFiles[absoluteConfig] = sha256File(absoluteConfig);
+ delete require.cache[require.resolve(absoluteConfig)];
+ let config;
+ try {
+ config = validateExtractionConfig(require(absoluteConfig));
+ } catch (error) {
+ throw new ExtractionError(error.message, 'input');
+ }
+ if (options.expectedLibDir) {
+ const configuredLibDir = fs.realpathSync(path.resolve(config.libDir));
+ const expectedLibDir = fs.realpathSync(path.resolve(options.expectedLibDir));
+ if (configuredLibDir !== expectedLibDir) {
+ throw new ExtractionError(`Extraction config libDir does not match report library: ${config.libDir}`, 'input');
+ }
+ }
+ if (options.expectedLibId && config.libId !== options.expectedLibId) {
+ throw new ExtractionError(`Extraction config libId does not match Data Pack meta.id: ${config.libId}`, 'input');
+ }
+ const enginePath = path.resolve(config.libDir, config.engineFile);
+ const engineBytes = fs.readFileSync(enginePath);
+ consumedFiles[enginePath] = sha256Bytes(engineBytes);
+ const engineSource = engineBytes.toString('utf8');
+ const defaults = sliceLiterals(engineSource, config.literals, config.libDir, consumedFiles);
+ const pack = config.buildPack(defaults);
+ if (!pack || typeof pack !== 'object' || Array.isArray(pack)) throw new Error('Extraction config buildPack must return an object');
+ pack.schemaVersion = config.schemaVersion || '1.3';
+ pack.meta = Object.assign({ id: config.libId, generatedBy: 'sg-data-pack: ' + path.basename(absoluteConfig) }, config.meta || {}, pack.meta || {});
+ const declaredAssets = pack.assets && typeof pack.assets === 'object' && !Array.isArray(pack.assets) ? pack.assets : {};
+ const scannedAssets = collectAssets(pack, config.libDir, consumedFiles, config.assetDir || 'lib/assets');
+ pack.assets = Object.fromEntries([...new Set([...Object.keys(declaredAssets), ...Object.keys(scannedAssets)])].map((assetPath) => [
+ assetPath,
+ /^(https?:|data:)/i.test(assetPath)
+ ? declaredAssets[assetPath]
+ : Object.assign({}, declaredAssets[assetPath] || {}, scannedAssets[assetPath] || { exists: false }),
+ ]));
+ const validation = globalThis.SGDataLoader.validate(pack);
+ const domain = typeof config.domainChecks === 'function' ? (config.domainChecks(pack) || {}) : {};
+ const domainErrors = Array.isArray(domain) ? domain : domain.errors || [];
+ const domainWarnings = Array.isArray(domain) ? [] : domain.warnings || [];
+ validation.errors.push(...domainErrors.map((item) => '[domain] ' + item));
+ validation.warnings.push(...domainWarnings.map((item) => '[domain] ' + item));
+ let equivalence = { comparisons: 0, diffs: [], coverage: equivalenceCoverage(config) };
+ if (!validation.errors.length) equivalence = runEquivalence(config, defaults, pack);
+ for (const moduleFile of Object.keys(require.cache)) {
+ if (modulesBefore.has(moduleFile) || !fs.existsSync(moduleFile) || !fs.statSync(moduleFile).isFile()) continue;
+ consumedFiles[path.resolve(moduleFile)] = sha256File(moduleFile);
+ }
+ return {
+ configPath: absoluteConfig,
+ config,
+ defaults,
+ pack,
+ validation,
+ equivalence,
+ consumedFiles,
+ inventory: {
+ literals: Object.fromEntries(Object.entries(defaults).map(([key, value]) => [key, Array.isArray(value) ? value.length : Object.keys(value || {}).length])),
+ assets: Object.keys(pack.assets || {}).length,
+ },
+ };
+}
+
+function runExtraction(configPath, options = {}) {
+ try {
+ return runExtractionUnsafe(configPath, options);
+ } catch (error) {
+ if (error instanceof ExtractionError) throw error;
+ const inputLike = error && ['ENOENT', 'EACCES', 'EISDIR', 'MODULE_NOT_FOUND', 'SyntaxError'].includes(error.code || error.name);
+ throw new ExtractionError(error.message, inputLike ? 'input' : 'gate');
+ }
+}
+
+module.exports = {
+ ExtractionError,
+ sliceLiteral,
+ sliceLiterals,
+ deepEqual,
+ resolveAssetRoot,
+ localAssetPath,
+ collectAssets,
+ equivalenceCoverage,
+ runEquivalence,
+ validateExtractionConfig,
+ runExtraction,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-extract-report-worker.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-extract-report-worker.js
new file mode 100644
index 0000000..1eacfef
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-extract-report-worker.js
@@ -0,0 +1,30 @@
+'use strict';
+
+/* Execute untrusted extraction config/engine code away from report stdout. */
+const fs = require('node:fs');
+const { runExtraction } = require('./sg-pack-extract-core.js');
+
+const [, , configPath, expectedLibDir, expectedLibId] = process.argv;
+let payload;
+try {
+ const result = runExtraction(configPath, { expectedLibDir, expectedLibId });
+ payload = {
+ ok: true,
+ result: {
+ inventory: result.inventory,
+ validation: result.validation,
+ equivalence: result.equivalence,
+ consumedFiles: result.consumedFiles,
+ },
+ };
+} catch (error) {
+ payload = {
+ ok: false,
+ error: {
+ name: error.name,
+ message: error.message,
+ kind: error.kind || 'gate',
+ },
+ };
+}
+fs.writeFileSync(3, JSON.stringify(payload));
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-inspect.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-inspect.js
new file mode 100644
index 0000000..099e9e3
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-inspect.js
@@ -0,0 +1,159 @@
+'use strict';
+
+/* Read-only Data Pack inventory, validation, and local asset integrity collector. */
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+require('./sg-data-loader.js');
+
+const loader = globalThis.SGDataLoader;
+
+function sha256Bytes(bytes) {
+ return 'sha256:' + crypto.createHash('sha256').update(bytes).digest('hex');
+}
+
+function sha1File(file) {
+ return 'sha1:' + crypto.createHash('sha1').update(fs.readFileSync(file)).digest('hex');
+}
+
+function sha256File(file) {
+ return 'sha256:' + crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+}
+
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function countObject(value) {
+ return isObject(value) ? Object.keys(value).length : 0;
+}
+
+function packInventory(pack) {
+ return {
+ entities: countObject(pack.entities),
+ aliases: countObject(pack.aliases),
+ relationTypes: countObject(pack.relationTypes),
+ heroRelTypes: countObject(pack.heroRelTypes),
+ relations: Array.isArray(pack.relations) ? pack.relations.length : 0,
+ stages: Array.isArray(pack.stages) ? pack.stages.length : 0,
+ contents: countObject(pack.contents),
+ domainKeys: countObject(pack.domain),
+ assets: countObject(pack.assets),
+ sameAs: Array.isArray(pack.sameAs) ? pack.sameAs.length : 0,
+ provenanceEntities: countObject(pack.provenance && pack.provenance.entities),
+ derivations: countObject(pack.derivations),
+ };
+}
+
+function readPack(file) {
+ const bytes = fs.readFileSync(file);
+ return { bytes, digest: sha256Bytes(bytes), pack: JSON.parse(bytes.toString('utf8')) };
+}
+
+function assetFilePath(dataFile, assetKey, assetRoot, assetBase = '') {
+ if (typeof assetKey !== 'string' || !assetKey || /[\u0000-\u001f\u007f]/.test(assetKey)) {
+ throw new Error('unsafe asset path');
+ }
+ const root = assetRoot
+ ? path.resolve(assetRoot)
+ : path.resolve(path.dirname(dataFile), '..', 'assets');
+ const normalizedKey = assetKey.replace(/\\/g, '/');
+ const normalizedBase = typeof assetBase === 'string' ? assetBase.replace(/\\/g, '/') : '';
+ const relative = normalizedBase && normalizedKey.startsWith(normalizedBase)
+ ? normalizedKey.slice(normalizedBase.length)
+ : normalizedKey.replace(/^(\.\.\/)?assets\//, '');
+ if (path.isAbsolute(relative) || /^[A-Za-z]:[\\/]/.test(relative)) throw new Error('asset path must be relative');
+ const resolved = path.resolve(root, relative);
+ const fromRoot = path.relative(root, resolved);
+ if (fromRoot === '..' || fromRoot.startsWith(`..${path.sep}`) || path.isAbsolute(fromRoot)) {
+ throw new Error('asset path escapes the asset root');
+ }
+ const rootStat = fs.lstatSync(root);
+ if (rootStat.isSymbolicLink()) throw new Error('asset root must not be a symlink');
+ let current = root;
+ for (const segment of relative.split(path.sep)) {
+ current = path.join(current, segment);
+ if (fs.existsSync(current) && fs.lstatSync(current).isSymbolicLink()) throw new Error('asset path contains a symlink component');
+ }
+ const physicalRoot = fs.realpathSync(root);
+ const physicalFile = fs.realpathSync(resolved);
+ const physicalRelative = path.relative(physicalRoot, physicalFile);
+ if (physicalRelative === '..' || physicalRelative.startsWith(`..${path.sep}`) || path.isAbsolute(physicalRelative)) {
+ throw new Error('asset symlink escapes the asset root');
+ }
+ return physicalFile;
+}
+
+function inspectAssets(pack, dataFile, options = {}) {
+ const consumedFiles = options.consumedFiles || null;
+ const checked = [];
+ const missing = [];
+ const mismatches = [];
+ const skipped = [];
+ if (!options.verifyHash) {
+ return {
+ enabled: false,
+ checked,
+ missing,
+ mismatches,
+ skipped: Object.keys(isObject(pack.assets) ? pack.assets : {}).map((assetKey) => ({ path: assetKey, reason: 'verification-not-requested' })),
+ summary: { checked: 0, passed: 0, missing: 0, mismatches: 0, skipped: Object.keys(isObject(pack.assets) ? pack.assets : {}).length },
+ };
+ }
+ const assets = isObject(pack.assets) ? pack.assets : {};
+ for (const [assetKey, manifest] of Object.entries(assets)) {
+ if (!manifest || !manifest.hash) {
+ skipped.push({ path: assetKey, reason: 'missing-manifest-hash' });
+ continue;
+ }
+ if (/^(https?:|data:)/i.test(assetKey)) {
+ skipped.push({ path: assetKey, reason: 'remote-or-data-uri' });
+ continue;
+ }
+ let file = null;
+ try {
+ file = assetFilePath(dataFile, assetKey, options.assetRoot, pack.meta && pack.meta.assetBase);
+ if (consumedFiles) consumedFiles[file] = sha256File(file);
+ const actualHash = sha1File(file);
+ const item = { path: assetKey, file, expected: manifest.hash, actual: actualHash, status: actualHash === manifest.hash ? 'passed' : 'mismatch' };
+ checked.push(item);
+ if (actualHash !== manifest.hash) mismatches.push(item);
+ } catch (error) {
+ const item = { path: assetKey, file, expected: manifest.hash, status: 'missing', error: error.message };
+ missing.push(item);
+ }
+ }
+ return {
+ enabled: Boolean(options.verifyHash),
+ checked,
+ missing,
+ mismatches,
+ skipped,
+ summary: {
+ checked: checked.length,
+ passed: checked.filter((item) => item.status === 'passed').length,
+ missing: missing.length,
+ mismatches: mismatches.length,
+ skipped: skipped.length,
+ },
+ };
+}
+
+function inspectPack({ dataFile, verifyHash = false, assetRoot } = {}) {
+ if (!dataFile) throw new Error('dataFile is required');
+ const loaded = readPack(dataFile);
+ const validation = loader.validate(loaded.pack);
+ const consumedFiles = { [path.resolve(dataFile)]: loaded.digest };
+ const assets = inspectAssets(loaded.pack, dataFile, { verifyHash, assetRoot, consumedFiles });
+ return {
+ dataFile: path.resolve(dataFile),
+ digest: loaded.digest,
+ pack: loaded.pack,
+ inventory: packInventory(loaded.pack),
+ validation,
+ assets,
+ consumedFiles,
+ };
+}
+
+module.exports = { sha256Bytes, packInventory, readPack, assetFilePath, inspectAssets, inspectPack };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-rules-core.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-rules-core.js
new file mode 100644
index 0000000..3787fdc
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-pack-rules-core.js
@@ -0,0 +1,140 @@
+'use strict';
+
+/* Structured, reusable data-rules evaluator. The legacy CLI renders this result. */
+
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function validateRulesDocument(rulesDoc) {
+ const errors = [];
+ if (!isObject(rulesDoc)) return ['rules document must be an object'];
+ if (rulesDoc.rulesVersion !== '1.0') errors.push('rulesVersion must be "1.0"');
+ if (typeof rulesDoc.libId !== 'string' || !rulesDoc.libId) errors.push('libId must be a non-empty string');
+ if (!isObject(rulesDoc.profile)) errors.push('profile is required');
+ if (!Array.isArray(rulesDoc.rules) || !rulesDoc.rules.length) errors.push('rules must be a non-empty array');
+ const ids = new Set();
+ (Array.isArray(rulesDoc.rules) ? rulesDoc.rules : []).forEach((rule, index) => {
+ const at = `rules[${index}]`;
+ if (!isObject(rule)) {
+ errors.push(`${at} must be an object`);
+ return;
+ }
+ for (const field of ['id', 'level', 'subject', 'rule', 'source']) {
+ if (typeof rule[field] !== 'string' || !rule[field]) errors.push(`${at}.${field} must be a non-empty string`);
+ }
+ if (rule.level && !['hard', 'soft'].includes(rule.level)) errors.push(`${at}.level must be "hard" or "soft"`);
+ if (rule.source && !['observed', 'extracted', 'human'].includes(rule.source)) errors.push(`${at}.source must be observed/extracted/human`);
+ if (rule.check !== undefined && rule.check !== null && typeof rule.check !== 'string') errors.push(`${at}.check must be a JS expression string or null`);
+ if (rule.id) {
+ if (ids.has(rule.id)) errors.push(`${at}.id "${rule.id}" is duplicated`);
+ ids.add(rule.id);
+ }
+ });
+ return errors;
+}
+
+function emptyCounts() {
+ return { total: 0, passed: 0, failed: 0, errors: 0, noCheck: 0, hardFail: 0, softFail: 0 };
+}
+
+function executeRules(rulesDoc, pack, options = {}) {
+ /* Validate the complete document before applying a selection. A malformed
+ * unselected rule must never disappear behind --rule. */
+ const structuralErrors = validateRulesDocument(rulesDoc);
+ if (structuralErrors.length) {
+ return {
+ libId: rulesDoc && rulesDoc.libId,
+ structuralErrors,
+ unknownRuleIds: [],
+ results: [],
+ counts: emptyCounts(),
+ passed: false,
+ };
+ }
+ const requestedRuleIds = options.ruleIds === undefined ? null : options.ruleIds;
+ if (requestedRuleIds !== null && (!Array.isArray(requestedRuleIds) || requestedRuleIds.some(id => typeof id !== 'string' || !id))) {
+ return {
+ libId: rulesDoc.libId,
+ profile: rulesDoc.profile,
+ structuralErrors: [],
+ selectionErrors: ['ruleIds must be an array of non-empty strings'],
+ unknownRuleIds: [],
+ results: [],
+ counts: emptyCounts(),
+ passed: false,
+ };
+ }
+ const requestedSet = requestedRuleIds === null ? null : new Set(requestedRuleIds);
+ const knownIds = new Set(rulesDoc.rules.map(rule => rule.id));
+ const unknownRuleIds = requestedRuleIds === null ? [] : [...requestedSet].filter(id => !knownIds.has(id));
+ if (unknownRuleIds.length) {
+ return {
+ libId: rulesDoc.libId,
+ profile: rulesDoc.profile,
+ structuralErrors: [],
+ selectionErrors: [],
+ unknownRuleIds,
+ results: [],
+ counts: emptyCounts(),
+ passed: false,
+ };
+ }
+ const selectedRules = requestedSet === null ? rulesDoc.rules : rulesDoc.rules.filter(rule => requestedSet.has(rule.id));
+ pack = pack || {};
+ const scope = {
+ entities: pack.entities || {}, relations: pack.relations || [], stages: pack.stages || [],
+ contents: pack.contents || {}, domain: pack.domain || {}, assets: pack.assets || {},
+ aliases: pack.aliases || {}, relationTypes: pack.relationTypes || {}, heroRelTypes: pack.heroRelTypes || {},
+ attributeTypes: pack.attributeTypes || {}, attributeSources: pack.attributeSources || [],
+ kindNameFields: pack.kindNameFields || {}, meta: pack.meta || {}, sameAs: pack.sameAs || [],
+ provenance: pack.provenance || {}, derivations: pack.derivations || {}, pack,
+ };
+ const keys = Object.keys(scope);
+ const results = [];
+ for (const rule of selectedRules) {
+ const base = {
+ id: rule.id,
+ level: rule.level,
+ subject: rule.subject,
+ rule: rule.rule,
+ rationale: rule.rationale || null,
+ evidence: rule.evidence || null,
+ source: rule.source,
+ repairHint: rule.repairHint || null,
+ coveredBy: rule.coveredBy === undefined ? null : rule.coveredBy,
+ };
+ if (!rule.check) {
+ results.push({ ...base, status: 'no-check', passed: null });
+ continue;
+ }
+ try {
+ const fn = new Function(...keys, '"use strict"; return (' + rule.check + ');');
+ const passed = Boolean(fn(...keys.map((key) => scope[key])));
+ results.push({ ...base, status: passed ? 'passed' : 'failed', passed });
+ } catch (error) {
+ results.push({ ...base, status: 'error', passed: false, exception: { name: error.name, message: error.message } });
+ }
+ }
+ const counts = {
+ total: results.length,
+ passed: results.filter((item) => item.status === 'passed').length,
+ failed: results.filter((item) => item.status === 'failed').length,
+ errors: results.filter((item) => item.status === 'error').length,
+ noCheck: results.filter((item) => item.status === 'no-check').length,
+ hardFail: results.filter((item) => item.passed === false && item.level === 'hard').length,
+ softFail: results.filter((item) => item.passed === false && item.level === 'soft').length,
+ };
+ return {
+ libId: rulesDoc.libId,
+ profile: rulesDoc.profile,
+ structuralErrors: [],
+ selectionErrors: [],
+ unknownRuleIds: [],
+ results,
+ counts,
+ passed: counts.hardFail === 0 && counts.errors === 0,
+ };
+}
+
+module.exports = { validateRulesDocument, executeRules };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-patch-executor.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-patch-executor.js
new file mode 100644
index 0000000..eee7a52
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-patch-executor.js
@@ -0,0 +1,255 @@
+'use strict';
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { runCommand } = require('./sg-command-runner.js');
+const { safeRelativePath, matchesGlob, assertNoSymlinkComponents } = require('./sg-path-policy.js');
+const { snapshotTree, diffTrees } = require('./sg-tree-snapshot.js');
+const { sha256Bytes, contentId } = require('./sg-evidence-utils.js');
+
+class PatchError extends Error {
+ constructor(message, kind = 'patch-invalid', details = null) {
+ super(message);
+ this.name = 'PatchError';
+ this.kind = kind;
+ this.details = details;
+ }
+}
+
+function stripPatchPrefix(value) {
+ if (value === '/dev/null') return null;
+ const withoutTimestamp = value.split('\t')[0].trim();
+ return withoutTimestamp.replace(/^[ab]\//, '');
+}
+
+function gitFileMode(statMode) {
+ return '100' + (statMode & 0o777).toString(8).padStart(3, '0');
+}
+
+function parseModeHeaders(lines, from, to) {
+ const modes = {};
+ for (let index = from; index < to; index += 1) {
+ const match = /^(new file mode|deleted file mode) ([0-7]{6})$/.exec(lines[index]);
+ if (!match) continue;
+ const field = match[1] === 'new file mode' ? 'newFileMode' : 'deletedFileMode';
+ if (modes[field]) throw new PatchError(`duplicate ${match[1]} header at line ${index + 1}`);
+ modes[field] = match[2];
+ }
+ if (modes.newFileMode && modes.newFileMode !== '100644') throw new PatchError(`new file mode ${modes.newFileMode} is not supported`, 'policy-not-supported');
+ return modes;
+}
+
+function parseUnifiedDiff(bytes) {
+ const text = Buffer.isBuffer(bytes) ? bytes.toString('utf8') : String(bytes);
+ if (text.includes('\0')) throw new PatchError('patch contains a NUL byte');
+ if (/^(GIT binary patch|Binary files .* differ)$/m.test(text)) throw new PatchError('binary patches are not supported', 'policy-not-supported');
+ if (/^(rename from|rename to|old mode|new mode|new file mode 160000|deleted file mode 160000)/m.test(text)) throw new PatchError('renames, mode changes, and submodules are not supported', 'policy-not-supported');
+ if (!text.startsWith('diff --git ')) throw new PatchError('patch must begin with a diff --git header');
+ const lines = text.split(/\r?\n/);
+ const operations = [];
+ let pendingGit = null;
+ for (let index = 0; index < lines.length; index += 1) {
+ const line = lines[index];
+ if (line.startsWith('diff --git ')) {
+ const match = /^diff --git a\/(.+) b\/(.+)$/.exec(line);
+ if (!match) throw new PatchError(`unsupported diff header at line ${index + 1}`);
+ pendingGit = { oldPath: match[1], newPath: match[2], line: index + 1 };
+ continue;
+ }
+ if (!line.startsWith('--- ')) continue;
+ if (index + 1 >= lines.length || !lines[index + 1].startsWith('+++ ')) throw new PatchError(`missing +++ header after line ${index + 1}`);
+ if (!pendingGit) throw new PatchError(`file header is missing its diff --git header at line ${index + 1}`);
+ const modes = parseModeHeaders(lines, pendingGit.line, index);
+ const oldPath = stripPatchPrefix(line.slice(4));
+ const newPath = stripPatchPrefix(lines[index + 1].slice(4));
+ if (!oldPath && !newPath) throw new PatchError(`invalid /dev/null pair at line ${index + 1}`);
+ if (oldPath && newPath && oldPath !== newPath) throw new PatchError('renames are not supported', 'policy-not-supported', { oldPath, newPath });
+ const target = safeRelativePath(newPath || oldPath);
+ if (pendingGit) {
+ if (safeRelativePath(pendingGit.oldPath) !== (oldPath || target) || safeRelativePath(pendingGit.newPath) !== (newPath || target)) {
+ throw new PatchError(`diff --git and file headers disagree near line ${index + 1}`);
+ }
+ }
+ const operation = oldPath === null ? 'add' : newPath === null ? 'delete' : 'modify';
+ if (operation === 'add' && modes.deletedFileMode) throw new PatchError(`add entry has a deleted file mode header: ${target}`);
+ if (operation === 'delete' && modes.newFileMode) throw new PatchError(`delete entry has a new file mode header: ${target}`);
+ if (operation === 'modify' && (modes.newFileMode || modes.deletedFileMode)) throw new PatchError(`modify entry must not contain add/delete mode headers: ${target}`);
+ let hunks = 0;
+ let cursor = index + 2;
+ while (cursor < lines.length && !lines[cursor].startsWith('diff --git ') && !lines[cursor].startsWith('--- ')) {
+ if (lines[cursor].startsWith('@@ ')) hunks += 1;
+ cursor += 1;
+ }
+ if (hunks === 0) throw new PatchError(`patch entry has no hunks: ${target}`);
+ operations.push({ operation, path: target, oldPath, newPath, hunks, modes, headerLine: index + 1 });
+ pendingGit = null;
+ }
+ if (!operations.length) throw new PatchError('patch contains no unified diff file entries');
+ const seen = new Set();
+ for (const operation of operations) {
+ if (seen.has(operation.path)) throw new PatchError(`patch contains duplicate file entries: ${operation.path}`);
+ seen.add(operation.path);
+ }
+ return { text, operations };
+}
+
+function policyForPath(filePolicy, operation, relativePath) {
+ const forbidden = (filePolicy.forbidden || []).find((pattern) => matchesGlob(relativePath, pattern));
+ if (forbidden) return { allowed: false, reason: `forbidden by ${forbidden}`, deniedBy: forbidden, allowedBy: null };
+ const allowed = (filePolicy.allowed || []).find((entry) => entry.operations.includes(operation) && matchesGlob(relativePath, entry.pattern));
+ if (!allowed) return { allowed: false, reason: `no allowed ${operation} rule matches ${relativePath}`, deniedBy: null, allowedBy: null };
+ return { allowed: true, reason: null, deniedBy: null, allowedBy: allowed.pattern };
+}
+
+function validatePatchPolicy(parsed, patchBytes, manifest, workspaceRoot) {
+ const violations = [];
+ if (patchBytes.length > manifest.filePolicy.maxPatchBytes) violations.push(`patch bytes ${patchBytes.length} exceed maxPatchBytes ${manifest.filePolicy.maxPatchBytes}`);
+ if (parsed.operations.length > manifest.filePolicy.maxChangedFiles) violations.push(`patch changes ${parsed.operations.length} files; maxChangedFiles is ${manifest.filePolicy.maxChangedFiles}`);
+ for (const operation of parsed.operations) {
+ const policy = policyForPath(manifest.filePolicy, operation.operation, operation.path);
+ if (operation.operation === 'add' && operation.modes.newFileMode !== '100644') violations.push(`add ${operation.path} must declare new file mode 100644`);
+ if (operation.operation === 'delete' && !operation.modes.deletedFileMode) violations.push(`delete ${operation.path} must declare its deleted file mode`);
+ operation.policy = policy;
+ if (!policy.allowed) violations.push(`${operation.operation} ${operation.path}: ${policy.reason}`);
+ const target = path.resolve(workspaceRoot, operation.path);
+ try { assertNoSymlinkComponents(target, { root: workspaceRoot }); }
+ catch (error) { violations.push(`${operation.path}: ${error.message}`); }
+ if (operation.operation === 'add' && fs.existsSync(target)) violations.push(`add target already exists: ${operation.path}`);
+ if (operation.operation !== 'add' && !fs.existsSync(target)) violations.push(`${operation.operation} target does not exist: ${operation.path}`);
+ if (fs.existsSync(target)) {
+ const stat = fs.lstatSync(target);
+ if (!stat.isFile() || stat.isSymbolicLink()) violations.push(`patch target must be a regular non-symlink file: ${operation.path}`);
+ if (manifest.filePolicy.allowHardlinks === false && stat.nlink > 1) violations.push(`hardlinked patch target is forbidden: ${operation.path}`);
+ if (operation.operation === 'delete' && operation.modes.deletedFileMode && gitFileMode(stat.mode) !== operation.modes.deletedFileMode) violations.push(`delete mode for ${operation.path} does not match baseline: expected ${gitFileMode(stat.mode)}, received ${operation.modes.deletedFileMode}`);
+ }
+ }
+ return { allowed: violations.length === 0, violations, operations: parsed.operations };
+}
+
+function validateAppliedDiff(diff, manifest) {
+ const violations = [];
+ const operations = [];
+ for (const change of diff.changes) {
+ const structuralEntry = change.after || change.before;
+ if (structuralEntry && structuralEntry.kind === 'directory' && ['add', 'delete'].includes(change.change)) continue;
+ const operation = change.change;
+ if (!['add', 'modify', 'delete'].includes(operation)) {
+ violations.push(`${operation} is not supported for ${change.path}`);
+ operations.push({ operation, path: change.path, allowed: false, reason: 'operation not supported' });
+ continue;
+ }
+ const entry = operation === 'delete' ? change.before : change.after;
+ if (entry && entry.kind !== 'file') violations.push(`${operation} produced non-regular entry: ${change.path}`);
+ if (operation === 'add' && entry && entry.mode !== 0o644) violations.push(`add produced unsupported mode ${entry.mode.toString(8)}: ${change.path}`);
+ if (operation === 'modify' && change.before && change.after && change.before.mode !== change.after.mode) violations.push(`modify changed file mode: ${change.path}`);
+ const policy = policyForPath(manifest.filePolicy, operation, change.path);
+ operations.push({ operation, path: change.path, allowed: policy.allowed, reason: policy.reason, before: change.before || null, after: change.after || null });
+ if (!policy.allowed) violations.push(`${operation} ${change.path}: ${policy.reason}`);
+ }
+ if (operations.length > manifest.filePolicy.maxChangedFiles) violations.push(`actual tree changes ${operations.length} files; maxChangedFiles is ${manifest.filePolicy.maxChangedFiles}`);
+ return { allowed: violations.length === 0, violations, operations };
+}
+
+function capturePatchTargets(root, operations) {
+ const originals = new Map();
+ for (const operation of operations) {
+ const target = path.join(root, operation.path);
+ if (!fs.existsSync(target)) originals.set(operation.path, { existed: false });
+ else {
+ const stat = fs.lstatSync(target);
+ originals.set(operation.path, { existed: true, bytes: fs.readFileSync(target), mode: stat.mode & 0o7777 });
+ }
+ }
+ return originals;
+}
+
+function restorePatchTargets(root, originals) {
+ const errors = [];
+ for (const [relative, original] of originals.entries()) {
+ const target = path.join(root, relative);
+ try {
+ if (!original.existed) fs.rmSync(target, { recursive: true, force: true });
+ else {
+ fs.mkdirSync(path.dirname(target), { recursive: true });
+ fs.writeFileSync(target, original.bytes, { mode: original.mode });
+ fs.chmodSync(target, original.mode);
+ }
+ } catch (error) { errors.push(`${relative}: ${error.message}`); }
+ }
+ if (errors.length) throw new Error('patch rollback incomplete: ' + errors.join('; '));
+}
+
+function applyPatch({ manifest, workspaceRoot, patchBytes, taskId = manifest && manifest.taskId, timeoutMs = 120000 } = {}) {
+ if (!manifest || !manifest.filePolicy || !manifest.patchPolicy) throw new PatchError('validated task manifest is required', 'input-error');
+ if (manifest.patchPolicy.format !== 'unified-diff' || manifest.patchPolicy.fuzz !== 0 || manifest.patchPolicy.allowBinary !== false) throw new PatchError('unsupported patch policy', 'policy-not-supported');
+ const root = fs.realpathSync(path.resolve(workspaceRoot));
+ const bytes = Buffer.isBuffer(patchBytes) ? patchBytes : Buffer.from(String(patchBytes || ''), 'utf8');
+ const before = snapshotTree(root, { errorOnSpecialFile: true, exclude: false });
+ let parsed;
+ let preflight;
+ try {
+ parsed = parseUnifiedDiff(bytes);
+ preflight = validatePatchPolicy(parsed, bytes, manifest, root);
+ if (!preflight.allowed) throw new PatchError('patch violates file policy', 'policy-violation', preflight.violations);
+ } catch (error) {
+ const audit = buildAudit({ taskId, bytes, before, parsed, preflight, status: 'rejected', failure: error });
+ error.audit = audit;
+ throw error;
+ }
+ const originals = capturePatchTargets(root, parsed.operations);
+ const patchFile = path.join(os.tmpdir(), `sg-agent-patch-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.diff`);
+ fs.writeFileSync(patchFile, bytes, { mode: 0o600, flag: 'wx' });
+ let checkRun;
+ let applyRun;
+ try {
+ checkRun = runCommand({ argv: ['git', 'apply', '--check', '--whitespace=error-all', patchFile], cwd: root, allowedRoot: root, timeoutMs, maxOutputBytes: 1048576 });
+ if (checkRun.error || checkRun.exitCode !== 0) throw new PatchError('git apply --check rejected the patch', 'patch-invalid', { checkRun });
+ applyRun = runCommand({ argv: ['git', 'apply', '--whitespace=error-all', patchFile], cwd: root, allowedRoot: root, timeoutMs, maxOutputBytes: 1048576 });
+ if (applyRun.error || applyRun.exitCode !== 0) throw new PatchError('git apply failed after a successful check', 'infra-error', { applyRun });
+ const after = snapshotTree(root, { errorOnSpecialFile: true, exclude: false });
+ const diff = diffTrees(before, after, { detectRenames: true });
+ const postflight = validateAppliedDiff(diff, manifest);
+ const declared = parsed.operations.map((item) => `${item.operation}:${item.path}`).sort();
+ const actual = postflight.operations.map((item) => `${item.operation}:${item.path}`).sort();
+ if (JSON.stringify(declared) !== JSON.stringify(actual)) postflight.violations.push(`declared patch operations do not match actual tree changes; declared=${declared.join(',')} actual=${actual.join(',')}`);
+ postflight.allowed = postflight.violations.length === 0;
+ if (!postflight.allowed) throw new PatchError('applied patch violates postflight file policy', 'policy-violation', postflight.violations);
+ const audit = buildAudit({ taskId, bytes, before, after, parsed, preflight, postflight, checkRun, applyRun, status: 'applied' });
+ return { audit, before, after, diff, parsed };
+ } catch (error) {
+ const current = snapshotTree(root, { errorOnSpecialFile: true, exclude: false });
+ if (current.treeSha256 !== before.treeSha256) restorePatchTargets(root, originals);
+ const restored = snapshotTree(root, { errorOnSpecialFile: true, exclude: false });
+ if (restored.treeSha256 !== before.treeSha256) throw new PatchError('patch rollback did not restore the workspace tree', 'infra-error', { before: before.treeSha256, restored: restored.treeSha256 });
+ const audit = buildAudit({ taskId, bytes, before, parsed, preflight, checkRun, applyRun, status: 'rejected', failure: error });
+ error.audit = audit;
+ throw error;
+ } finally {
+ try { fs.unlinkSync(patchFile); } catch (_) { /* already absent */ }
+ }
+}
+
+function summarizeRun(run) {
+ if (!run) return null;
+ return { argv: run.argv, cwd: run.cwd, exitCode: run.exitCode, signal: run.signal, timedOut: run.timedOut, durationMs: run.durationMs, stdoutSha256: sha256Bytes(run.stdout || ''), stderrSha256: sha256Bytes(run.stderr || ''), error: run.error || null };
+}
+
+function buildAudit({ taskId, bytes, before, after = null, parsed = null, preflight = null, postflight = null, checkRun = null, applyRun = null, status, failure = null }) {
+ const audit = {
+ auditVersion: '1.0', auditId: null, kind: 'patch-audit', taskId,
+ status, patchSha256: sha256Bytes(bytes), patchBytes: bytes.length,
+ beforeTreeSha256: 'sha256:' + before.treeSha256,
+ afterTreeSha256: after ? 'sha256:' + after.treeSha256 : null,
+ declaredOperations: parsed ? parsed.operations.map((item) => ({ operation: item.operation, path: item.path, hunks: item.hunks, modes: item.modes, policy: item.policy || null })) : [],
+ actualOperations: postflight ? postflight.operations : [],
+ preflight: preflight ? { allowed: preflight.allowed, violations: preflight.violations } : null,
+ postflight: postflight ? { allowed: postflight.allowed, violations: postflight.violations } : null,
+ commands: { check: summarizeRun(checkRun), apply: summarizeRun(applyRun) },
+ failure: failure ? { kind: failure.kind || 'infra-error', name: failure.name, message: failure.message, details: failure.details || null } : null,
+ capabilities: { osSandbox: false, networkIsolation: false, processIsolation: false, sourceTreeVerifiedByRunner: true, disposableWorkspace: true },
+ };
+ audit.auditId = contentId('patch-audit', audit, ['auditId']);
+ return audit;
+}
+
+module.exports = { PatchError, parseUnifiedDiff, policyForPath, validatePatchPolicy, validateAppliedDiff, applyPatch, buildAudit };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-path-policy.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-path-policy.js
new file mode 100644
index 0000000..78b738e
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-path-policy.js
@@ -0,0 +1,311 @@
+'use strict';
+
+/*
+ * Small, dependency-free filesystem policy primitives used by repository tools.
+ * All paths are treated as untrusted input until they have passed the relevant
+ * checks below. The module deliberately does not create, remove, or mutate
+ * files.
+ */
+const fs = require('node:fs');
+const path = require('node:path');
+
+const CONTROL_PATH = /[\u0000-\u001f\u007f]/;
+const WINDOWS_ABSOLUTE = /^[A-Za-z]:/;
+
+function pathError(code, message, details) {
+ const error = new Error(message);
+ error.code = code;
+ if (details !== undefined) error.details = details;
+ return error;
+}
+
+function assertPathString(value, name = 'path') {
+ if (typeof value !== 'string' || value.length === 0) {
+ throw pathError('INVALID_PATH', `${name} must be a non-empty string`);
+ }
+ if (CONTROL_PATH.test(value)) {
+ throw pathError('INVALID_PATH', `${name} contains a control character`);
+ }
+}
+
+function canonicalPath(value, options = {}) {
+ assertPathString(value);
+ const absolute = path.resolve(value);
+ const mustExist = options.mustExist === true;
+ try {
+ return fs.realpathSync.native(absolute);
+ } catch (error) {
+ if (mustExist || (error && error.code !== 'ENOENT' && error.code !== 'ENOTDIR')) throw error;
+
+ // realpathSync cannot resolve a path which does not exist yet. Resolve the
+ // nearest existing ancestor, then append the non-existent suffix. This
+ // still canonicalizes symlinked parents and makes containment checks safe
+ // for output paths.
+ const suffix = [];
+ let current = absolute;
+ while (true) {
+ try {
+ const physical = fs.realpathSync.native(current);
+ return path.join(physical, ...suffix.reverse());
+ } catch (ancestorError) {
+ if (ancestorError && (ancestorError.code === 'ENOENT' || ancestorError.code === 'ENOTDIR')) {
+ const parent = path.dirname(current);
+ if (parent === current) throw ancestorError;
+ suffix.push(path.basename(current));
+ current = parent;
+ continue;
+ }
+ throw ancestorError;
+ }
+ }
+ }
+}
+
+function sameFile(left, right, options = {}) {
+ const leftPath = canonicalPath(left, options);
+ const rightPath = canonicalPath(right, options);
+ try {
+ // canonical paths cover symlink aliases; device/inode also covers hard
+ // links, which are the other common meaning of “same file”. A missing
+ // path is not a file, even when both arguments spell the same path.
+ const leftStat = fs.statSync(leftPath);
+ const rightStat = fs.statSync(rightPath);
+ return leftPath === rightPath || (leftStat.dev === rightStat.dev && leftStat.ino === rightStat.ino);
+ } catch (error) {
+ if (options.mustExist) throw error;
+ return false;
+ }
+}
+
+function safeRelativePath(value, options = {}) {
+ if (value === '' && options.allowEmpty) return '';
+ assertPathString(value, 'relative path');
+ if (path.isAbsolute(value) || WINDOWS_ABSOLUTE.test(value) || value.startsWith('\\')) {
+ throw pathError('UNSAFE_RELATIVE_PATH', `path must be relative: ${value}`);
+ }
+ // Treat both separators as separators. This prevents a Windows-style
+ // traversal from becoming a harmless filename when checked on POSIX.
+ const pieces = value.replace(/\\/g, '/').split('/');
+ if (pieces.some((piece) => piece === '..')) {
+ throw pathError('UNSAFE_RELATIVE_PATH', `path contains a parent traversal: ${value}`);
+ }
+ if (pieces.length === 1 && pieces[0] === '.' && options.allowEmpty) return '';
+ if (pieces.some((piece) => piece === '' || piece === '.')) {
+ throw pathError('UNSAFE_RELATIVE_PATH', `path contains an empty or dot segment: ${value}`);
+ }
+ return pieces.join('/');
+}
+
+function isSafeRelativePath(value, options) {
+ try {
+ safeRelativePath(value, options);
+ return true;
+ } catch (_) {
+ return false;
+ }
+}
+
+function isWithinRoot(root, target, options = {}) {
+ const physicalRoot = canonicalPath(root, options);
+ const physicalTarget = canonicalPath(target, options);
+ const relative = path.relative(physicalRoot, physicalTarget);
+ if (relative === '') return options.allowRoot !== false;
+ return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+}
+
+function assertWithinRoot(root, target, options = {}) {
+ if (!isWithinRoot(root, target, options)) {
+ throw pathError('PATH_OUTSIDE_ROOT', `path is outside allowed root: ${target}`, {
+ root: canonicalPath(root, options),
+ target: canonicalPath(target, options),
+ });
+ }
+ return canonicalPath(target, options);
+}
+
+function existingPathComponents(value) {
+ assertPathString(value);
+ const absolute = path.resolve(value);
+ const components = [];
+ let current = absolute;
+ while (true) {
+ components.push(current);
+ const parent = path.dirname(current);
+ if (parent === current) break;
+ current = parent;
+ }
+ return components.reverse();
+}
+
+function symlinkComponent(value, options = {}) {
+ const components = existingPathComponents(value);
+ let startIndex = 0;
+ if (options.root) {
+ const root = path.resolve(options.root);
+ const rootIndex = components.indexOf(root);
+ if (rootIndex !== -1) startIndex = rootIndex;
+ }
+ if (options.root && options.includeRoot === false) startIndex += 1;
+ for (let index = startIndex; index < components.length; index += 1) {
+ const component = components[index];
+ try {
+ if (fs.lstatSync(component).isSymbolicLink()) return component;
+ } catch (error) {
+ if (error.code !== 'ENOENT' && error.code !== 'ENOTDIR') throw error;
+ }
+ }
+ return null;
+}
+
+function hasSymlinkComponent(value, options) {
+ return symlinkComponent(value, options) !== null;
+}
+
+function assertNoSymlinkComponents(value, options) {
+ const component = symlinkComponent(value, options);
+ if (component) {
+ throw pathError('SYMLINK_COMPONENT', `path contains a symlink component: ${value}`, { component });
+ }
+ return true;
+}
+
+function globPatternRegex(pattern) {
+ assertPathString(pattern, 'glob pattern');
+ const normalized = pattern.replace(/\\/g, '/');
+ let source = '^';
+ for (let index = 0; index < normalized.length; index += 1) {
+ const character = normalized[index];
+ if (character === '*' && normalized[index + 1] === '*') {
+ if (normalized[index + 2] === '/') {
+ source += '(?:.*/)?';
+ index += 2;
+ } else if (index + 2 === normalized.length || normalized[index + 2] !== '/') {
+ // A trailing globstar also matches the directory itself, as common
+ // ignore-file glob syntax does (foo/** matches foo and foo/child).
+ if (index > 0 && normalized[index - 1] === '/') {
+ source = source.slice(0, -1) + '(?:/.*)?';
+ } else {
+ source += '.*';
+ }
+ index += 1;
+ } else {
+ source += '.*';
+ index += 1;
+ }
+ } else if (character === '*') {
+ source += '[^/]*';
+ } else if (character === '?') {
+ // '?' is intentionally literal: the policy contract only has * and **.
+ source += '\\?';
+ } else {
+ source += character.replace(/[.+^${}()|[\\]\\\\]/g, '\\\\$&');
+ }
+ }
+ return new RegExp(`${source}$`);
+}
+
+function matchesGlob(value, pattern) {
+ const relative = safeRelativePath(value, { allowEmpty: true });
+ return globPatternRegex(pattern).test(relative);
+}
+
+function asPatterns(value) {
+ if (value === undefined || value === null) return [];
+ if (typeof value === 'string') return [value];
+ if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) {
+ throw pathError('INVALID_GLOB_POLICY', 'allow/deny patterns must be strings or arrays of strings');
+ }
+ return value;
+}
+
+function evaluateGlobPolicy(value, policy = {}) {
+ const relative = safeRelativePath(value, { allowEmpty: true });
+ const deny = asPatterns(policy.deny || policy.denied);
+ const allow = asPatterns(policy.allow || policy.allowed);
+ const deniedBy = deny.find((pattern) => matchesGlob(relative || '.', pattern));
+ if (deniedBy !== undefined) return { allowed: false, path: relative, deniedBy, allowedBy: null };
+ if (allow.length === 0) return { allowed: true, path: relative, deniedBy: null, allowedBy: null };
+ const allowedBy = allow.find((pattern) => matchesGlob(relative || '.', pattern));
+ return { allowed: allowedBy !== undefined, path: relative, deniedBy: null, allowedBy: allowedBy || null };
+}
+
+function isPathAllowed(value, policy) {
+ return evaluateGlobPolicy(value, policy).allowed;
+}
+
+function operationPolicy(config = {}) {
+ const root = config.root || config.allowedRoot || null;
+ const operations = config.operations || config.operation || {};
+
+ function check(operation, target, options = {}) {
+ const selected = operations[operation] || config[operation] || config;
+ const selectedRoot = options.root || root;
+ let relative = target;
+ let canonicalTarget = null;
+ if (selectedRoot) {
+ canonicalTarget = assertWithinRoot(selectedRoot, target, {
+ allowRoot: options.allowRoot,
+ mustExist: options.mustExist,
+ });
+ relative = path.relative(canonicalPath(selectedRoot), canonicalTarget).split(path.sep).join('/');
+ }
+ const globResult = evaluateGlobPolicy(relative, selected || {});
+ return {
+ ...globResult,
+ operation,
+ target,
+ canonicalTarget,
+ root: selectedRoot,
+ };
+ }
+
+ function assert(operation, target, options = {}) {
+ const result = check(operation, target, options);
+ if (!result.allowed) {
+ const reason = result.deniedBy ? `denied by ${result.deniedBy}` : 'not included by allow policy';
+ throw pathError('OPERATION_NOT_ALLOWED', `${operation} is not allowed for ${target}: ${reason}`, result);
+ }
+ return result;
+ }
+
+ return {
+ check,
+ evaluate: check,
+ isAllowed: (operation, target, options) => check(operation, target, options).allowed,
+ assert,
+ assertAllowed: assert,
+ };
+}
+
+function checkOperation(operation, target, config) {
+ return operationPolicy(config).check(operation, target);
+}
+
+function assertOperationAllowed(operation, target, config) {
+ return operationPolicy(config).assert(operation, target);
+}
+
+module.exports = {
+ canonicalPath,
+ sameFile,
+ safeRelativePath,
+ assertSafeRelativePath: safeRelativePath,
+ isSafeRelativePath,
+ isWithinRoot,
+ isPathWithinRoot: isWithinRoot,
+ assertWithinRoot,
+ symlinkComponent,
+ hasSymlinkComponent,
+ hasSymlinkComponents: hasSymlinkComponent,
+ assertNoSymlinkComponent: assertNoSymlinkComponents,
+ assertNoSymlinkComponents,
+ matchesGlob,
+ globMatch: matchesGlob,
+ evaluateGlobPolicy,
+ isPathAllowed,
+ pathAllowed: isPathAllowed,
+ operationPolicy,
+ createOperationPolicy: operationPolicy,
+ checkOperation,
+ assertOperationAllowed,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-recrawl-review.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-recrawl-review.js
new file mode 100644
index 0000000..5164754
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-recrawl-review.js
@@ -0,0 +1,388 @@
+'use strict';
+
+/*
+ * Pure helpers shared by recrawl-skeleton and candidate-pack.
+ * No filesystem, process, or clock access belongs in this module.
+ */
+const crypto = require('node:crypto');
+require('./sg-data-loader.js');
+
+const OBSERVATION_VERSION = '1.0';
+const REVIEWABLE = new Set(['gap', 'conflict', 'unsupported']);
+const OBSERVATION_META_FIELDS = new Set(['sourceUrl', 'origin', 'fetchedAt', 'confidence']);
+
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function sha256(value) {
+ const bytes = Buffer.isBuffer(value) ? value : Buffer.from(String(value), 'utf8');
+ return 'sha256:' + crypto.createHash('sha256').update(bytes).digest('hex');
+}
+
+function stableJson(value) {
+ return JSON.stringify(value);
+}
+
+function normalizeRecords(rawRecords) {
+ if (!Array.isArray(rawRecords)) throw new Error('records.json must contain an array of names or record objects');
+ return rawRecords.map((record, inputIndex) => {
+ if (typeof record === 'string') {
+ if (!record.trim()) throw new Error(`records[${inputIndex}] must contain a non-empty name`);
+ return {
+ inputIndex,
+ crawledName: record.trim(),
+ fields: {},
+ raw: record,
+ };
+ }
+ if (!isObject(record)) throw new Error(`records[${inputIndex}] must be a string or object`);
+ const { crawledName, name, ...fields } = record;
+ const resolvedName = crawledName || name;
+ if (typeof resolvedName !== 'string' || !resolvedName.trim()) {
+ throw new Error(`records[${inputIndex}] must contain crawledName or name`);
+ }
+ return {
+ inputIndex,
+ crawledName: resolvedName.trim(),
+ fields,
+ raw: record,
+ };
+ });
+}
+
+function resolveResolution(pack, name) {
+ if (isObject(pack.entities) && Object.prototype.hasOwnProperty.call(pack.entities, name)) {
+ return { status: 'hit', entityId: name, via: 'id' };
+ }
+ const alias = isObject(pack.aliases) ? pack.aliases[name] : undefined;
+ if (alias === undefined) return { status: 'miss', via: 'none' };
+ const entityId = isObject(alias) ? alias.id : alias;
+ if (entityId && isObject(pack.entities) && Object.prototype.hasOwnProperty.call(pack.entities, entityId)) {
+ const resolution = { status: 'hit', entityId, via: 'alias' };
+ if (isObject(alias) && alias.context !== undefined) resolution.aliasContext = alias.context;
+ return resolution;
+ }
+ return { status: 'miss', via: 'invalid-alias' };
+}
+
+function normalizeName(value) {
+ return String(value).toLowerCase().normalize('NFD').replace(/[\u0300-\u036f]/g, '').replace(/[\s·・\-_.'"()()]+/g, '');
+}
+
+function levenshtein(a, b) {
+ const m = a.length;
+ const n = b.length;
+ if (!m) return n;
+ if (!n) return m;
+ const dp = new Uint16Array(n + 1);
+ for (let j = 0; j <= n; j += 1) dp[j] = j;
+ for (let i = 1; i <= m; i += 1) {
+ let previous = dp[0];
+ dp[0] = i;
+ for (let j = 1; j <= n; j += 1) {
+ const saved = dp[j];
+ dp[j] = Math.min(
+ dp[j] + 1,
+ dp[j - 1] + 1,
+ previous + (a[i - 1] === b[j - 1] ? 0 : 1),
+ );
+ previous = saved;
+ }
+ }
+ return dp[n];
+}
+
+function pairScore(a, b) {
+ const left = normalizeName(a);
+ const right = normalizeName(b);
+ if (!left || !right) return 0;
+ if (left === right) return 1;
+ if (left.startsWith(right) || right.startsWith(left)) {
+ return 0.6 + 0.35 * (Math.min(left.length, right.length) / Math.max(left.length, right.length));
+ }
+ if (left.includes(right) || right.includes(left)) {
+ return Math.min(left.length, right.length) / Math.max(left.length, right.length) * 0.95;
+ }
+ return 1 - levenshtein(left, right) / Math.max(left.length, right.length);
+}
+
+function similarity(a, b) {
+ const full = pairScore(a, b);
+ const leftTokens = String(a).split(/[\s·・\-_]+/).filter(Boolean);
+ const rightTokens = String(b).split(/[\s·・\-_]+/).filter(Boolean);
+ let best = 0;
+ for (const left of leftTokens) for (const right of rightTokens) best = Math.max(best, pairScore(left, right));
+ return Math.max(full, best * 0.9);
+}
+
+function mappedField(pack, field) {
+ const fieldMap = pack.meta && pack.meta.recrawlFieldMap;
+ if (isObject(fieldMap) && Object.prototype.hasOwnProperty.call(fieldMap, field) && typeof fieldMap[field] === 'string' && fieldMap[field]) {
+ return fieldMap[field];
+ }
+ return field;
+}
+
+function nameField(pack, id, entity) {
+ const fields = pack.kindNameFields;
+ const field = isObject(fields) && Object.prototype.hasOwnProperty.call(fields, entity.kind) ? fields[entity.kind] : null;
+ return (typeof field === 'string' && entity[field]) || entity.name || id;
+}
+
+function buildSurfaces(pack) {
+ const surfaces = [];
+ for (const [id, entity] of Object.entries(pack.entities || {})) {
+ surfaces.push({ id, text: id, via: 'id' });
+ const display = nameField(pack, id, entity);
+ if (display && display !== id) surfaces.push({ id, text: display, via: 'name' });
+ }
+ for (const [alias, target] of Object.entries(pack.aliases || {})) {
+ const id = target && typeof target === 'object' ? target.id : target;
+ if (id && (pack.entities || {})[id]) surfaces.push({ id, text: alias, via: 'alias' });
+ }
+ return surfaces;
+}
+
+function candidatesFor(pack, name, topN = 3, stable = true) {
+ const scored = new Map();
+ for (const surface of buildSurfaces(pack)) {
+ const score = similarity(name, surface.text);
+ const current = scored.get(surface.id);
+ if (!current || score > current.score || (stable && score === current.score && surface.text < current.text)) {
+ scored.set(surface.id, { score, via: surface.via, text: surface.text });
+ }
+ }
+ return [...scored.entries()]
+ .map(([id, value]) => ({ id, score: +value.score.toFixed(3), matchedOn: `${value.via}:"${value.text}"`, _text: value.text }))
+ .filter((candidate) => candidate.score >= 0.5)
+ .sort((a, b) => stable
+ ? (b.score - a.score || a.id.localeCompare(b.id) || a._text.localeCompare(b._text))
+ : b.score - a.score)
+ .slice(0, topN)
+ .map(({ _text, ...candidate }) => candidate);
+}
+
+function isCalendarDate(value) {
+ if (!/^\d{4}-\d{2}-\d{2}$/.test(value || '')) return false;
+ const date = new Date(`${value}T00:00:00Z`);
+ return date.toISOString().slice(0, 10) === value;
+}
+
+function isHttpUrl(value) {
+ if (typeof value !== 'string' || /[\u0000-\u001f\u007f]/.test(value)) return false;
+ try {
+ const url = new URL(value);
+ return (url.protocol === 'http:' || url.protocol === 'https:') && Boolean(url.hostname);
+ } catch (_) {
+ return false;
+ }
+}
+
+function isEmptyBaseline(value) {
+ return value === undefined || value === null || value === '' || value === '待定';
+}
+
+function isIgnoredField(field) {
+ return field === 'crawledName' || field === 'name';
+}
+
+function legacyCrossCheck(pack, id, fields) {
+ const entity = pack.entities && pack.entities[id];
+ if (!entity) return { status: 'no-baseline' };
+ const result = { agree: [], conflict: [], gap: [] };
+ for (const [field, crawled] of Object.entries(fields)) {
+ if (isIgnoredField(field) || crawled === undefined || crawled === null || crawled === '') continue;
+ const baselineField = mappedField(pack, field);
+ const hasBaseline = Object.prototype.hasOwnProperty.call(entity, baselineField);
+ const baseline = hasBaseline ? entity[baselineField] : undefined;
+ if (isEmptyBaseline(baseline)) {
+ result.gap.push({ field, baselineField, baseline, crawled });
+ } else {
+ const baselineNormalized = String(baseline).replace(/ 饰$/, '').trim();
+ const crawledNormalized = String(crawled).replace(/ 饰$/, '').trim();
+ if (baselineNormalized === crawledNormalized) result.agree.push(field);
+ else result.conflict.push({ field, baselineField, baseline, crawled });
+ }
+ }
+ return result;
+}
+
+function checkValue(pack, id, field, crawled) {
+ const entity = pack.entities[id];
+ const baselineField = mappedField(pack, field);
+ const baselineExists = Object.prototype.hasOwnProperty.call(entity, baselineField);
+ const baseline = baselineExists ? entity[baselineField] : undefined;
+ const base = {
+ crawledField: field,
+ baselineField,
+ baselineExists,
+ baseline,
+ crawled,
+ };
+ if (OBSERVATION_META_FIELDS.has(field)) return { ...base, classification: 'unsupported', reason: 'crawl metadata is provenance, not an entity field' };
+ if (field.includes('.') || field.includes('[') || field.includes(']')) {
+ return { ...base, classification: 'unsupported', reason: 'nested field paths are not supported' };
+ }
+ if (field === '__proto__' || field === 'prototype' || field === 'constructor') {
+ return { ...base, classification: 'unsupported', reason: 'reserved object key' };
+ }
+ if (typeof crawled !== 'string' || crawled === '') {
+ return { ...base, classification: 'unsupported', reason: 'only non-empty string observations are supported' };
+ }
+ if (typeof baseline !== 'undefined' && baseline !== null && typeof baseline !== 'string') {
+ return { ...base, classification: 'unsupported', reason: 'baseline field is not a string' };
+ }
+ if (isEmptyBaseline(baseline)) return { ...base, classification: 'gap' };
+ const baselineNormalized = baseline.replace(/ 饰$/, '').trim();
+ const crawledNormalized = crawled.replace(/ 饰$/, '').trim();
+ return {
+ ...base,
+ classification: baselineNormalized === crawledNormalized ? 'agree' : 'conflict',
+ };
+}
+
+function recordId(record) {
+ const material = stableJson({ inputIndex: record.inputIndex, crawledName: record.crawledName, fields: record.fields });
+ return `r${String(record.inputIndex).padStart(6, '0')}-${sha256(material).slice(7, 19)}`;
+}
+
+function itemId(record, suffix) {
+ return `${recordId(record)}:${suffix}`;
+}
+
+function buildLegacyProjection(pack, rawRecords) {
+ const records = normalizeRecords(rawRecords);
+ const hits = [];
+ const misses = [];
+ for (const record of records) {
+ const resolution = resolveResolution(pack, record.crawledName);
+ if (resolution.status === 'hit') {
+ hits.push({
+ crawledName: record.crawledName,
+ id: resolution.entityId,
+ crossCheck: legacyCrossCheck(pack, resolution.entityId, record.fields),
+ });
+ } else {
+ misses.push({ crawledName: record.crawledName, candidates: candidatesFor(pack, record.crawledName, 3, false) });
+ }
+ }
+ return {
+ summary: { total: records.length, hits: hits.length, misses: misses.length },
+ hits,
+ misses,
+ autoMergeable: hits.filter((hit) => hit.crossCheck.gap && hit.crossCheck.gap.length).map((hit) => ({
+ id: hit.id,
+ crawledName: hit.crawledName,
+ gaps: hit.crossCheck.gap,
+ })),
+ needsHumanReview: hits.filter((hit) => hit.crossCheck.conflict && hit.crossCheck.conflict.length).map((hit) => ({
+ id: hit.id,
+ crawledName: hit.crawledName,
+ conflicts: hit.crossCheck.conflict,
+ })),
+ };
+}
+
+function buildCandidateReview({ pack, rawRecords, baselineBytes, recordsBytes, origin, sourceUrl, fetchedAt }) {
+ if (typeof origin !== 'string' || !/^crawl:[^\s]+$/.test(origin.trim())) {
+ throw new Error('candidate-ready mode requires --origin crawl:');
+ }
+ if (!isHttpUrl(sourceUrl)) throw new Error('candidate-ready mode requires an HTTP(S) --source URL with a hostname');
+ if (!isCalendarDate(fetchedAt)) throw new Error('candidate-ready mode requires a real --fetchedAt YYYY-MM-DD date');
+ const records = normalizeRecords(rawRecords);
+ const observations = [];
+ const reviewItems = [];
+ for (const record of records) {
+ const resolution = resolveResolution(pack, record.crawledName);
+ const observation = {
+ recordId: recordId(record),
+ inputIndex: record.inputIndex,
+ crawledName: record.crawledName,
+ rawFields: record.fields,
+ resolution,
+ checks: [],
+ };
+ if (resolution.status === 'hit') {
+ const legacy = legacyCrossCheck(pack, resolution.entityId, record.fields);
+ observation.checks = Object.entries(record.fields)
+ .filter(([field]) => !isIgnoredField(field) && !OBSERVATION_META_FIELDS.has(field))
+ .map(([field, value]) => ({
+ itemId: itemId(record, `field:${field}`),
+ ...checkValue(pack, resolution.entityId, field, value),
+ }));
+ // Keep the legacy calculation as the compatibility source of truth for its projection.
+ observation.legacyCrossCheck = legacy;
+ for (const check of observation.checks) {
+ if (REVIEWABLE.has(check.classification)) {
+ reviewItems.push({
+ itemId: check.itemId,
+ kind: 'field',
+ recordId: observation.recordId,
+ inputIndex: record.inputIndex,
+ entityId: resolution.entityId,
+ classification: check.classification,
+ crawledField: check.crawledField,
+ baselineField: check.baselineField,
+ reason: check.reason,
+ });
+ }
+ }
+ } else {
+ observation.candidates = candidatesFor(pack, record.crawledName);
+ observation.checks = [];
+ reviewItems.push({
+ itemId: itemId(record, 'identity'),
+ kind: 'identity',
+ recordId: observation.recordId,
+ inputIndex: record.inputIndex,
+ crawledName: record.crawledName,
+ classification: 'miss',
+ });
+ }
+ observations.push(observation);
+ }
+ const candidateReview = {
+ version: OBSERVATION_VERSION,
+ baselineSha256: sha256(baselineBytes),
+ recordsSha256: sha256(recordsBytes),
+ origin: origin.trim(),
+ sourceUrl,
+ fetchedAt,
+ observations,
+ reviewItems,
+ };
+ candidateReview.reportId = sha256(stableJson(candidateReview));
+ return candidateReview;
+}
+
+function verifyCandidateReview(report, baselineBytes, recordsBytes) {
+ const candidateReview = report && report.candidateReview;
+ if (!isObject(candidateReview) || candidateReview.version !== OBSERVATION_VERSION) {
+ throw new Error('report is not candidate-ready; rerun recrawl-skeleton with --candidate-ready');
+ }
+ if (candidateReview.baselineSha256 !== sha256(baselineBytes)) throw new Error('baseline SHA-256 does not match candidate review report');
+ if (recordsBytes !== undefined && candidateReview.recordsSha256 !== sha256(recordsBytes)) {
+ throw new Error('records SHA-256 does not match candidate review report');
+ }
+ const { reportId, ...withoutId } = candidateReview;
+ if (reportId !== sha256(stableJson(withoutId))) throw new Error('candidate review reportId does not match report contents');
+ return candidateReview;
+}
+
+module.exports = {
+ OBSERVATION_VERSION,
+ OBSERVATION_META_FIELDS,
+ sha256,
+ stableJson,
+ normalizeRecords,
+ resolveResolution,
+ candidatesFor,
+ legacyCrossCheck,
+ checkValue,
+ recordId,
+ itemId,
+ buildLegacyProjection,
+ buildCandidateReview,
+ verifyCandidateReview,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-report-renderer.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-report-renderer.js
new file mode 100644
index 0000000..1d6aa99
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-report-renderer.js
@@ -0,0 +1,166 @@
+'use strict';
+
+/* Render a RunReport; never performs checks or filesystem writes. */
+
+function statusMark(status) {
+ return { passed: '✓', failed: '✖', 'not-assessed': '○', 'not-applicable': '—', 'review-required': '!' }[status] || '•';
+}
+
+function outcomeLabel(outcome) {
+ return {
+ ready: 'DATA-VALID',
+ 'issues-found': 'ISSUES-FOUND',
+ 'review-required': 'REVIEW-REQUIRED',
+ blocked: 'BLOCKED',
+ 'input-error': 'INPUT-ERROR',
+ }[outcome] || String(outcome || 'UNKNOWN').toUpperCase();
+}
+
+function short(value, max = 240) {
+ const text = typeof value === 'string' ? value : JSON.stringify(value);
+ if (!text) return '';
+ return text.length > max ? text.slice(0, max - 1) + '…' : text;
+}
+
+function renderJson(report) {
+ return JSON.stringify(report, null, 2) + '\n';
+}
+
+function renderTerminalSummary(report, options = {}) {
+ const lines = [];
+ lines.push(`SG Data Pack — ${report.run.libId}`);
+ lines.push(`状态:${outcomeLabel(report.run.outcome)} | 成熟度:${report.run.maturity}`);
+ lines.push(`摘要:${report.summary.findings} findings / ${report.summary.changes} changes / ${report.summary.notAssessed} 项未评估`);
+ lines.push('');
+
+ const section = (title, entries, formatter, emptyText) => {
+ lines.push(title);
+ if (!entries.length) lines.push(` ${emptyText}`);
+ else entries.slice(0, options.limit || 20).forEach((entry, index) => lines.push(formatter(entry, index)));
+ if (entries.length > (options.limit || 20)) lines.push(` … 其余 ${entries.length - (options.limit || 20)} 项见完整报告`);
+ lines.push('');
+ };
+
+ section('发现的问题', report.findings.filter((item) => ['open', 'review-required'].includes(item.status)), (item) => {
+ const subject = item.subject ? ` [${item.subject}]` : '';
+ const hint = item.repairHint ? `;建议:${short(item.repairHint, 150)}` : '';
+ return ` ${item.severity === 'error' ? '✖' : item.status === 'review-required' ? '!' : '⚠'} ${item.code}${subject}:${short(item.message)}${hint}`;
+ }, '未发现开放问题');
+
+ section('已修改 / 已完善', report.changes, (item) => {
+ const subject = item.subject ? ` [${item.subject}]` : '';
+ const transition = item.before !== null || item.after !== null
+ ? `:${short(item.before, 100)} → ${short(item.after, 100)}`
+ : '';
+ return ` • ${item.title || item.id}${subject}${transition}`;
+ }, '本次没有可比较的变更');
+
+ section('验证范围', report.assurances, (item) => {
+ const evidence = item.evidence ? ` — ${short(item.evidence, 160)}` : '';
+ return ` ${statusMark(item.status)} ${item.title}${evidence}`;
+ }, '没有验证项');
+
+ section('剩余风险', report.risks, (item) => {
+ const reason = item.reason || item.message || item.title || '';
+ return ` ⚠ ${item.title || item.id}:${short(reason)}`;
+ }, '没有额外风险记录');
+
+ section('下一步', report.nextSteps, (item, index) => {
+ const command = item.command ? `\n 命令:${item.command}` : '';
+ const doneWhen = item.doneWhen ? `\n 完成条件:${item.doneWhen}` : '';
+ return ` ${index + 1}. [${item.priority || 'P2'}] ${item.title}${command}${doneWhen}`;
+ }, '没有自动生成的下一步');
+
+ if (report.operations.length) {
+ lines.push(`审核操作:${report.operations.length} 项;derivation 影响:${report.impacts.length} 项`);
+ lines.push('');
+ }
+ if (options.reportPath) lines.push(`完整报告:${options.reportPath}`);
+ return lines.join('\n') + '\n';
+}
+
+function renderMarkdown(report, options = {}) {
+ const lines = [];
+ lines.push(`# SG Data Pack 报告:${report.run.libId}`);
+ lines.push('');
+ lines.push(`- **状态**:${outcomeLabel(report.run.outcome)}`);
+ lines.push(`- **成熟度**:${report.run.maturity}`);
+ lines.push(`- **Run ID**:\`${report.run.id}\``);
+ lines.push('');
+
+ lines.push('## 执行摘要');
+ lines.push('');
+ lines.push(`发现 **${report.summary.findings}** 项,变更 **${report.summary.changes}** 项,开放问题 **${report.summary.open}** 项,未评估 **${report.summary.notAssessed}** 项。`);
+ lines.push('');
+
+ lines.push('## 数据概况');
+ lines.push('');
+ lines.push('| 数据集合 | 数量 |');
+ lines.push('|---|---:|');
+ const inventoryLabels = {
+ entities: 'Entities', aliases: 'Aliases', relationTypes: 'Relation types', heroRelTypes: 'Hero relation types',
+ relations: 'Relations', stages: 'Stages', contents: 'Contents', domainKeys: 'Domain keys', assets: 'Assets',
+ sameAs: 'sameAs pairs', provenanceEntities: 'Entity provenance', derivations: 'Derivations',
+ };
+ Object.entries(inventoryLabels).forEach(([key, label]) => lines.push(`| ${label} | ${report.inventory[key] === undefined ? 0 : report.inventory[key]} |`));
+ lines.push('');
+
+ const mdList = (heading, entries, render, empty) => {
+ lines.push(`## ${heading}`);
+ lines.push('');
+ if (!entries.length) lines.push(`> ${empty}`);
+ else entries.forEach((entry, index) => lines.push(render(entry, index)));
+ lines.push('');
+ };
+ mdList('发现的问题', report.findings.filter((item) => ['open', 'review-required'].includes(item.status)), (item) => {
+ const subject = item.subject ? `(${item.subject})` : '';
+ const hint = item.repairHint ? `\n - 修复建议:${item.repairHint}` : '';
+ return `- [ ] **${item.code}** ${item.title}${subject}:${item.message}${hint}`;
+ }, '未发现开放问题。');
+
+ const resolvedFindings = report.findings.filter((item) => item.status === 'resolved');
+ if (resolvedFindings.length) {
+ lines.push('## 已解决问题');
+ lines.push('');
+ resolvedFindings.forEach((item) => lines.push(`- [x] **${item.code}** ${item.title}:${item.message}`));
+ lines.push('');
+ }
+
+ mdList('已修改 / 已完善', report.changes, (item) => {
+ const subject = item.subject ? `(${item.subject})` : '';
+ const beforeAfter = item.before !== null || item.after !== null ? `:\`${short(item.before, 160)}\` → \`${short(item.after, 160)}\`` : '';
+ return `- **${item.title || item.id}**${subject}${beforeAfter}`;
+ }, '本次没有可比较的变更。');
+
+ mdList('验证与保证范围', report.assurances, (item) => {
+ const evidence = item.evidence ? ` — ${item.evidence}` : '';
+ return `- ${statusMark(item.status)} **${item.title}**${evidence}`;
+ }, '没有验证项。');
+
+ mdList('剩余风险', report.risks, (item) => `- **${item.title || item.id}**:${item.reason || item.message || ''}`, '没有额外风险记录。');
+
+ mdList('下一步开发动作', report.nextSteps, (item, index) => {
+ const details = [item.owner && `owner: ${item.owner}`, item.command && `命令: \`${item.command}\``, item.doneWhen && `完成条件: ${item.doneWhen}`].filter(Boolean).join(';');
+ return `${index + 1}. **[${item.priority || 'P2'}] ${item.title}**${details ? ` — ${details}` : ''}`;
+ }, '没有自动生成的下一步。');
+
+ if (report.operations.length) {
+ lines.push('## 审核操作');
+ lines.push('');
+ lines.push(`共 ${report.operations.length} 项操作,详见 audit 输入或 JSON report。`);
+ lines.push('');
+ }
+ if (report.impacts.length) {
+ lines.push('## Derivation 影响');
+ lines.push('');
+ report.impacts.forEach((item) => lines.push(`- **${item.name || item.id}**:${(item.triggers || []).join(', ')}`));
+ lines.push('');
+ }
+ if (options.generatedBy) {
+ lines.push(`_Generated by sg-data-pack report; ${options.generatedBy}_`);
+ lines.push('');
+ }
+ return lines.join('\n');
+}
+
+module.exports = { renderJson, renderTerminalSummary, renderMarkdown, outcomeLabel };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-run-report.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-run-report.js
new file mode 100644
index 0000000..0ced1e5
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-run-report.js
@@ -0,0 +1,239 @@
+'use strict';
+
+/* Pure RunReport v1.0 builder. It deliberately performs no filesystem or process I/O. */
+const crypto = require('node:crypto');
+
+const REPORT_VERSION = '1.0';
+const OUTCOMES = new Set(['ready', 'issues-found', 'review-required', 'blocked', 'input-error']);
+const MATURITIES = ['UNASSESSED', 'ASSESSED', 'DATA-VALID'];
+
+function stableValue(value) {
+ if (Array.isArray(value)) return value.map(stableValue);
+ if (value && typeof value === 'object') {
+ return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableValue(value[key])]));
+ }
+ return value;
+}
+
+function stableJson(value) {
+ return JSON.stringify(stableValue(value));
+}
+
+function sha256(value) {
+ const bytes = Buffer.isBuffer(value) ? value : Buffer.from(String(value), 'utf8');
+ return 'sha256:' + crypto.createHash('sha256').update(bytes).digest('hex');
+}
+
+function asArray(value) {
+ return Array.isArray(value) ? value : [];
+}
+
+function countBy(items, key, value) {
+ return items.filter((item) => item && item[key] === value).length;
+}
+
+function normalizeMessage(value) {
+ if (typeof value === 'string') return { message: value, raw: value };
+ if (value instanceof Error) return { message: value.message, raw: { name: value.name, message: value.message } };
+ return { message: String(value), raw: value };
+}
+
+function diagnosticFinding({ value, phase, category = 'contract', severity = 'error', status = 'open', blocking = severity === 'error', code, title, subject, evidence, expected, actual, repairHint }) {
+ const normalized = normalizeMessage(value);
+ const parsedCode = code || (/^(E\d+|W\d+):/.exec(normalized.message) || [])[1] || 'TOOL_DIAGNOSTIC';
+ const shortMessage = normalized.message.replace(/^(E\d+|W\d+):\s*/, '');
+ return {
+ code: parsedCode,
+ severity,
+ status,
+ category,
+ phase,
+ title: title || (parsedCode === 'TOOL_DIAGNOSTIC' ? '工具诊断' : parsedCode),
+ subject: subject || null,
+ message: shortMessage,
+ evidence: evidence || null,
+ expected: expected === undefined ? null : expected,
+ actual: actual === undefined ? null : actual,
+ repairHint: repairHint || null,
+ blocking: Boolean(blocking),
+ raw: normalized.raw,
+ needsNormalization: parsedCode === 'TOOL_DIAGNOSTIC',
+ };
+}
+
+function withFindingIds(findings) {
+ const counts = new Map();
+ return findings.map((finding) => {
+ const base = String(finding.code || 'TOOL_DIAGNOSTIC').replace(/[^A-Za-z0-9_-]+/g, '-');
+ const index = (counts.get(base) || 0) + 1;
+ counts.set(base, index);
+ return { id: `F-${base}-${index}`, ...finding };
+ });
+}
+
+function assurance(id, title, status, options = {}) {
+ return {
+ id,
+ title,
+ status,
+ evidence: options.evidence || null,
+ details: options.details || null,
+ blocking: options.blocking === undefined ? status === 'failed' : Boolean(options.blocking),
+ };
+}
+
+function change({ id, section, kind, title, subject, before, after, fields, evidence }) {
+ return {
+ id,
+ section,
+ kind: kind || 'updated',
+ title: title || `${section} ${kind || 'updated'}`,
+ subject: subject || null,
+ before: before === undefined ? null : before,
+ after: after === undefined ? null : after,
+ fields: fields || [],
+ evidence: evidence || null,
+ };
+}
+
+function inventory(pack) {
+ const countObject = (value) => value && typeof value === 'object' && !Array.isArray(value) ? Object.keys(value).length : 0;
+ return {
+ entities: countObject(pack.entities),
+ aliases: countObject(pack.aliases),
+ relationTypes: countObject(pack.relationTypes),
+ heroRelTypes: countObject(pack.heroRelTypes),
+ relations: asArray(pack.relations).length,
+ stages: asArray(pack.stages).length,
+ contents: countObject(pack.contents),
+ domainKeys: countObject(pack.domain),
+ assets: countObject(pack.assets),
+ sameAs: asArray(pack.sameAs).length,
+ provenanceEntities: countObject(pack.provenance && pack.provenance.entities),
+ derivations: countObject(pack.derivations),
+ };
+}
+
+function deriveMaturity(report) {
+ if (!report.assurances.length) return 'UNASSESSED';
+ const contract = report.assurances.find((item) => item.id === 'data-pack-contract');
+ if (contract && contract.status === 'passed') return 'DATA-VALID';
+ if (report.assurances.some((item) => item.status === 'passed')) return 'ASSESSED';
+ return 'UNASSESSED';
+}
+
+function deriveOutcome(report) {
+ if (report.inputError) return 'input-error';
+ if (report.findings.some((item) => item.blocking && item.status === 'open')) return 'blocked';
+ if (report.findings.some((item) => item.status === 'review-required')) return 'review-required';
+ if (report.findings.some((item) => item.severity === 'warning' && item.status === 'open')) return 'issues-found';
+ if (report.assurances.some((item) => item.status === 'failed')) return 'blocked';
+ return 'ready';
+}
+
+function buildSummary(report) {
+ return {
+ findings: report.findings.length,
+ errors: countBy(report.findings, 'severity', 'error'),
+ warnings: countBy(report.findings, 'severity', 'warning'),
+ open: report.findings.filter((item) => ['open', 'review-required'].includes(item.status)).length,
+ resolved: countBy(report.findings, 'status', 'resolved'),
+ notAssessed: report.assurances.filter((item) => item.status === 'not-assessed').length,
+ changes: report.changes.length,
+ nextSteps: report.nextSteps.length,
+ };
+}
+
+function buildRunId({ libId, mode, inputs, inventory: packInventory }) {
+ return sha256(stableJson({ libId, mode, inputs, inventory: packInventory }));
+}
+
+function exitCodeForOutcome(outcome) {
+ if (outcome === 'input-error') return 2;
+ if (['blocked', 'review-required'].includes(outcome)) return 1;
+ return 0;
+}
+
+function buildNextSteps(report) {
+ const steps = [];
+ const add = (id, priority, owner, title, reason, command, doneWhen, blocking = false) => {
+ if (!steps.some((step) => step.id === id)) steps.push({ id, priority, owner, title, reason, command, doneWhen, blocking });
+ };
+ if (report.findings.some((item) => item.status === 'review-required')) {
+ add('review-open-items', 'P0', 'reviewer', '处理待人工复核项', '存在 gap、conflict、unsupported 或 identity miss。', 'node "$SK" candidate ...', '每个 review item 都有明确 decision,Candidate audit 状态为 valid。', true);
+ }
+ if (report.findings.some((item) => item.severity === 'error' && item.status === 'open')) {
+ add('fix-blocking-findings', 'P0', 'component-developer', '修复阻断性数据问题', 'Data Pack contract 或规则存在阻断 finding。', 'node "$SK" validate --strict --verify-hash', 'errors 为 0,且 strict validation 通过。', true);
+ }
+ if (report.assurances.some((item) => item.id === 'extract-equivalence' && item.status === 'not-assessed')) {
+ add('run-equivalence', 'P1', 'component-developer', '运行引擎等价性校验', '当前报告没有 extract config,尚未证明 __fromPack 无损还原默认数据。', 'node "$SK" extract --check', '所有 deep comparisons passed。');
+ }
+ if (report.assurances.some((item) => item.id === 'runtime-mount' && item.status === 'not-assessed')) {
+ add('add-runtime-mount', 'P1', 'component-developer', '补充 runtime mount 回归', '当前只验证数据合同,没有验证真实 DOM mount。', '运行组件库 runtime mount test', '页面可 mount,控制台无 Data Pack 错误。');
+ }
+ if (report.assurances.some((item) => item.id === 'visual-regression' && item.status === 'not-assessed')) {
+ add('add-visual-regression', 'P2', 'component-developer', '补充视觉回归', '当前没有截图或浏览器视觉证据。', '运行 visual regression 测试', '关键场景截图 diff 通过。');
+ }
+ if (report.impacts.length) {
+ add('review-derivation-impact', 'P1', 'component-developer', '复核 derivation 影响面', '数据变化会影响引擎 consumer 或展示区域。', 'node "$SK" diff --json', '每个 impacted derivation 都完成相应回归。');
+ }
+ return steps;
+}
+
+function buildReport(input = {}) {
+ const pack = input.pack || {};
+ const libId = input.libId || (pack.meta && pack.meta.id) || 'unknown-library';
+ const mode = input.mode || 'pack';
+ const rawFindings = asArray(input.findings);
+ const findings = withFindingIds(rawFindings);
+ const report = {
+ reportVersion: REPORT_VERSION,
+ run: {
+ id: null,
+ command: input.command || 'report',
+ exitCode: null,
+ libId,
+ mode,
+ outcome: 'ready',
+ maturity: 'UNASSESSED',
+ },
+ inputs: input.inputs || {},
+ summary: null,
+ inventory: input.inventory || inventory(pack),
+ findings,
+ changes: asArray(input.changes),
+ assurances: asArray(input.assurances),
+ coverage: asArray(input.coverage && input.coverage.length ? input.coverage : input.assurances),
+ risks: asArray(input.risks),
+ nextSteps: [],
+ operations: asArray(input.operations),
+ impacts: asArray(input.impacts),
+ artifacts: asArray(input.artifacts),
+ raw: input.raw || {},
+ inputError: Boolean(input.inputError),
+ };
+ report.nextSteps = asArray(input.nextSteps).concat(buildNextSteps(report));
+ report.summary = buildSummary(report);
+ report.run.maturity = deriveMaturity(report);
+ report.run.outcome = deriveOutcome(report);
+ report.run.exitCode = exitCodeForOutcome(report.run.outcome);
+ report.run.id = buildRunId({ libId, mode, inputs: report.inputs, inventory: report.inventory });
+ delete report.inputError;
+ return report;
+}
+
+module.exports = {
+ REPORT_VERSION,
+ OUTCOMES,
+ MATURITIES,
+ sha256,
+ stableJson,
+ diagnosticFinding,
+ withFindingIds,
+ assurance,
+ change,
+ inventory,
+ buildSummary,
+ exitCodeForOutcome,
+ buildReport,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-runtime-evidence.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-runtime-evidence.js
new file mode 100644
index 0000000..301fa75
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-runtime-evidence.js
@@ -0,0 +1,52 @@
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { contentId, isDigest, verifyArtifacts } = require('./sg-evidence-utils.js');
+
+function validateRuntimeEvidence(evidence, options = {}) {
+ const errors = [];
+ if (!evidence || typeof evidence !== 'object' || Array.isArray(evidence)) return { valid: false, errors: ['runtime evidence must be an object'], status: 'not-assessed' };
+ if (evidence.evidenceVersion !== '1.0') errors.push('runtime evidenceVersion must be 1.0');
+ if (evidence.kind !== 'runtime-evidence') errors.push('runtime kind must be runtime-evidence');
+ if (typeof evidence.scenarioId !== 'string' || !evidence.scenarioId) errors.push('runtime scenarioId is required');
+ if (!isDigest(evidence.subjectTreeSha256)) errors.push('runtime subjectTreeSha256 must be sha256:');
+ if (!evidence.producer || typeof evidence.producer !== 'object' || typeof evidence.producer.name !== 'string' || typeof evidence.producer.version !== 'string') errors.push('runtime producer name/version are required');
+ if (!evidence.environment || typeof evidence.environment !== 'object') errors.push('runtime environment is required');
+ if (!Array.isArray(evidence.assertions)) errors.push('runtime assertions must be an array');
+ if (!Array.isArray(evidence.consoleErrors)) errors.push('runtime consoleErrors must be an array');
+ if (!Array.isArray(evidence.pageErrors)) errors.push('runtime pageErrors must be an array');
+ if (!Array.isArray(evidence.networkFailures)) errors.push('runtime networkFailures must be an array');
+ if (!Array.isArray(evidence.artifacts) || !evidence.artifacts.length) errors.push('runtime artifacts must be a non-empty array');
+ const assertionIds = new Set();
+ for (const [index, assertion] of (evidence.assertions || []).entries()) {
+ if (!assertion || typeof assertion !== 'object' || typeof assertion.id !== 'string' || !assertion.id || !['passed', 'failed'].includes(assertion.status)) errors.push(`runtime assertions[${index}] must have id and passed|failed status`);
+ else if (assertionIds.has(assertion.id)) errors.push(`runtime assertion id is duplicated: ${assertion.id}`);
+ else assertionIds.add(assertion.id);
+ }
+ verifyArtifacts(evidence.artifacts, options.baseDir, errors);
+ const expectedId = contentId('runtime', evidence, ['evidenceId']);
+ if (evidence.evidenceId !== expectedId) errors.push(`runtime evidenceId mismatch (expected ${expectedId})`);
+ const failures = (evidence.assertions || []).filter((item) => item.status === 'failed').length
+ + (evidence.consoleErrors || []).length
+ + (evidence.pageErrors || []).length
+ + (evidence.networkFailures || []).length;
+ const complete = errors.length === 0 && (evidence.assertions || []).length > 0;
+ return {
+ valid: errors.length === 0,
+ errors,
+ status: !complete ? 'not-assessed' : failures ? 'failed' : 'passed',
+ summary: {
+ assertions: (evidence.assertions || []).length,
+ passed: (evidence.assertions || []).filter((item) => item.status === 'passed').length,
+ failed: failures,
+ },
+ };
+}
+
+function readRuntimeEvidence(file) {
+ const evidence = JSON.parse(fs.readFileSync(file, 'utf8'));
+ const validation = validateRuntimeEvidence(evidence, { baseDir: path.dirname(path.resolve(file)) });
+ return { evidence, validation };
+}
+
+module.exports = { validateRuntimeEvidence, readRuntimeEvidence };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-task-runner.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-task-runner.js
new file mode 100644
index 0000000..8c578f6
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-task-runner.js
@@ -0,0 +1,477 @@
+'use strict';
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { readTaskManifest, validateTaskManifest, resolveSourceRoot, computeTreeSha256, sha256: taskSha256 } = require('./agent-task-manifest.js');
+const { runCommand } = require('./sg-command-runner.js');
+const { snapshotTree } = require('./sg-tree-snapshot.js');
+const { applyPatch, PatchError } = require('./sg-patch-executor.js');
+const { gradeWorkspace, validateGraderSpec } = require('./sg-grader.js');
+const { sha256Bytes, contentId, safeRelative } = require('./sg-evidence-utils.js');
+const { canonicalPath } = require('./sg-path-policy.js');
+
+function ensureOutsideSource(sourceRoot, artifactRoot) {
+ const source = fs.realpathSync(sourceRoot);
+ const output = path.resolve(artifactRoot);
+ const relative = path.relative(source, output);
+ if (relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative))) throw new Error('artifact root must be outside source root');
+}
+
+function writeAtomic(file, bytes) {
+ fs.mkdirSync(path.dirname(file), { recursive: true });
+ const temp = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`);
+ let fd;
+ try {
+ fd = fs.openSync(temp, 'wx', 0o600);
+ fs.writeFileSync(fd, bytes);
+ fs.fsyncSync(fd);
+ fs.closeSync(fd); fd = undefined;
+ fs.renameSync(temp, file);
+ } catch (error) {
+ if (fd !== undefined) fs.closeSync(fd);
+ try { fs.unlinkSync(temp); } catch (_) { /* absent */ }
+ throw error;
+ }
+}
+
+function artifact(root, id, file, mediaType) {
+ const bytes = fs.readFileSync(file);
+ return { id, path: path.relative(root, file).split(path.sep).join('/'), mediaType, bytes: bytes.length, sha256: sha256Bytes(bytes) };
+}
+
+function mediaTypeFor(file) {
+ if (file.endsWith('.json')) return 'application/json';
+ if (file.endsWith('.png')) return 'image/png';
+ if (file.endsWith('.html')) return 'text/html';
+ if (file.endsWith('.diff') || file.endsWith('.patch')) return 'text/x-diff';
+ if (file.endsWith('.md')) return 'text/markdown';
+ return 'text/plain';
+}
+
+function collectOutputArtifacts(root, current) {
+ const known = new Set(current.map((item) => item.path));
+ const files = [];
+ const unsupported = [];
+ function walk(directory) {
+ for (const name of fs.readdirSync(directory).sort()) {
+ const file = path.join(directory, name);
+ const relative = path.relative(root, file).split(path.sep).join('/');
+ const stat = fs.lstatSync(file);
+ if (stat.isSymbolicLink()) unsupported.push(`${relative}: symlink`);
+ else if (stat.isDirectory()) walk(file);
+ else if (stat.isFile() && stat.nlink > 1) unsupported.push(`${relative}: hardlink`);
+ else if (stat.isFile()) files.push(file);
+ else unsupported.push(`${relative}: unsupported filesystem entry`);
+ }
+ }
+ walk(root);
+ for (const file of files) {
+ const relative = path.relative(root, file).split(path.sep).join('/');
+ if (relative === 'task-run.json' || known.has(relative)) continue;
+ const id = 'generated-' + relative.replace(/[^A-Za-z0-9._-]+/g, '-');
+ current.push(artifact(root, id, file, mediaTypeFor(file)));
+ known.add(relative);
+ }
+ current.sort((left, right) => left.path.localeCompare(right.path));
+ return unsupported;
+}
+
+function replaceTokens(argv, values) {
+ return argv.map((argument) => Object.entries(values).reduce((value, [token, replacement]) => value.split(token).join(replacement), argument));
+}
+
+function agentResult(run) {
+ return {
+ provider: null, model: null, argv: run.argv, cwd: run.cwd,
+ exitCode: run.exitCode, signal: run.signal, timedOut: run.timedOut, durationMs: run.durationMs,
+ stdoutSha256: sha256Bytes(run.stdout || ''), stderrSha256: sha256Bytes(run.stderr || ''),
+ stdout: run.stdout, stderr: run.stderr,
+ stdoutTruncated: run.stdoutTruncated, stderrTruncated: run.stderrTruncated, error: run.error || null,
+ };
+}
+
+function loadGraderSpec(reference, taskDir) {
+ if (reference && typeof reference === 'object' && !Array.isArray(reference)) return { spec: reference, root: taskDir };
+ const relative = safeRelative(reference, 'grader spec path');
+ const file = path.resolve(taskDir, relative);
+ const fromRoot = path.relative(taskDir, file);
+ if (fromRoot === '..' || fromRoot.startsWith(`..${path.sep}`) || path.isAbsolute(fromRoot)) throw new Error('grader spec escapes task directory');
+ return { spec: JSON.parse(fs.readFileSync(file, 'utf8')), root: path.dirname(file), file };
+}
+
+function buildRun(report) {
+ const material = { ...report };
+ delete material.runId;
+ report.runId = contentId('task-run', material, []);
+ return report;
+}
+
+function verifyFileBinding(binding) {
+ try {
+ const stat = fs.lstatSync(binding.file);
+ if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('must be a regular non-symlink file');
+ if (stat.nlink > 1) throw new Error('hardlinks are not allowed');
+ const actual = taskSha256(fs.readFileSync(binding.file));
+ return actual === binding.sha256 ? null : `digest mismatch: expected ${binding.sha256}, received ${actual}`;
+ } catch (error) { return error.message; }
+}
+
+function integrityIssues({ absoluteTask, taskFileSha256, loadedGraders, externalBindings }) {
+ const issues = [];
+ const taskIssue = verifyFileBinding({ file: absoluteTask, sha256: taskFileSha256 });
+ if (taskIssue) issues.push(`task manifest: ${taskIssue}`);
+ for (const loaded of loadedGraders) {
+ const specIssue = verifyFileBinding({ file: loaded.file, sha256: loaded.reference.sha256 });
+ if (specIssue) issues.push(`grader ${loaded.reference.id} spec: ${specIssue}`);
+ try {
+ const actualTree = computeTreeSha256(loaded.root);
+ if (actualTree !== loaded.reference.treeSha256) issues.push(`grader ${loaded.reference.id} tree digest mismatch: expected ${loaded.reference.treeSha256}, received ${actualTree}`);
+ } catch (error) { issues.push(`grader ${loaded.reference.id} tree: ${error.message}`); }
+ }
+ for (const binding of externalBindings) {
+ const bindingIssue = verifyFileBinding(binding);
+ if (bindingIssue) issues.push(`${binding.id}: ${bindingIssue}`);
+ }
+ return issues;
+}
+
+function captureTrustedTools() {
+ const originalScriptsRoot = path.resolve(__dirname, '..');
+ const original = snapshotTree(originalScriptsRoot, { errorOnSpecialFile: true, exclude: false });
+ return { originalScriptsRoot, originalTreeSha256: original.treeSha256 };
+}
+
+function stageTrustedTools(parent, captured) {
+ const current = snapshotTree(captured.originalScriptsRoot, { errorOnSpecialFile: true, exclude: false });
+ if (current.treeSha256 !== captured.originalTreeSha256) throw new Error('original tool scripts changed before staging');
+ const toolRoot = path.join(parent, 'trusted-tools');
+ const scriptsRoot = path.join(toolRoot, 'scripts');
+ fs.mkdirSync(toolRoot, { recursive: true });
+ fs.cpSync(captured.originalScriptsRoot, scriptsRoot, { recursive: true, dereference: false, verbatimSymlinks: true });
+ const staged = snapshotTree(scriptsRoot, { errorOnSpecialFile: true, exclude: false });
+ if (staged.treeSha256 !== captured.originalTreeSha256) throw new Error('staged tool scripts tree digest mismatch');
+ return { ...captured, toolRoot, scriptsRoot, stagedTreeSha256: staged.treeSha256 };
+}
+
+function verifyTrustedTools(tools) {
+ const issues = [];
+ try {
+ const original = snapshotTree(tools.originalScriptsRoot, { errorOnSpecialFile: true, exclude: false });
+ if (original.treeSha256 !== tools.originalTreeSha256) issues.push(`original tool scripts changed: expected ${tools.originalTreeSha256}, received ${original.treeSha256}`);
+ } catch (error) { issues.push('original tool scripts: ' + error.message); }
+ if (tools.scriptsRoot) {
+ try {
+ const staged = snapshotTree(tools.scriptsRoot, { errorOnSpecialFile: true, exclude: false });
+ if (staged.treeSha256 !== tools.stagedTreeSha256) issues.push(`staged tool scripts changed: expected ${tools.stagedTreeSha256}, received ${staged.treeSha256}`);
+ } catch (error) { issues.push('staged tool scripts: ' + error.message); }
+ }
+ return issues;
+}
+
+function stageGraders(loadedGraders, parent) {
+ const stagingRoot = path.join(parent, 'trusted-graders');
+ fs.mkdirSync(stagingRoot, { recursive: true });
+ return loadedGraders.map((loaded) => {
+ const root = path.join(stagingRoot, loaded.reference.id);
+ fs.cpSync(loaded.root, root, { recursive: true, dereference: false, verbatimSymlinks: true });
+ const treeSha256 = computeTreeSha256(root);
+ if (treeSha256 !== loaded.reference.treeSha256) throw new Error(`staged grader ${loaded.reference.id} tree digest mismatch`);
+ const relativeSpec = path.relative(loaded.root, loaded.file);
+ const file = path.join(root, relativeSpec);
+ const specIssue = verifyFileBinding({ file, sha256: loaded.reference.sha256 });
+ if (specIssue) throw new Error(`staged grader ${loaded.reference.id} spec ${specIssue}`);
+ const spec = JSON.parse(fs.readFileSync(file, 'utf8'));
+ const validation = validateGraderSpec(spec);
+ if (!validation.valid) throw new Error(`staged grader ${loaded.reference.id} is invalid: ${validation.errors.join('; ')}`);
+ return { reference: loaded.reference, root, file, spec };
+ });
+}
+
+function runTask({ taskFile, agentArgv, artifactRoot, agent = {}, keepWorkspace = false, trialId = null, integrityInputs = [] } = {}) {
+ if (!taskFile || !Array.isArray(agentArgv) || !agentArgv.length || !artifactRoot) throw new Error('taskFile, non-empty agentArgv, and artifactRoot are required');
+ if (!Array.isArray(integrityInputs) || integrityInputs.some((binding) => !binding || typeof binding.id !== 'string' || typeof binding.file !== 'string' || typeof binding.sha256 !== 'string')) throw new TypeError('integrityInputs must contain id, file, and sha256 bindings');
+ const initialExternalIssues = integrityInputs.map((binding) => ({ binding, issue: verifyFileBinding(binding) })).filter((item) => item.issue);
+ if (initialExternalIssues.length) {
+ const error = new Error('task integrity input validation failed: ' + initialExternalIssues.map((item) => `${item.binding.id}: ${item.issue}`).join('; '));
+ error.kind = 'input-error';
+ throw error;
+ }
+ const infraExitCodes = agent.infraExitCodes === undefined ? [] : agent.infraExitCodes;
+ if (!Array.isArray(infraExitCodes) || infraExitCodes.some((value) => !Number.isInteger(value) || value < 1 || value > 255) || new Set(infraExitCodes).size !== infraExitCodes.length) throw new TypeError('agent.infraExitCodes must contain unique exit codes from 1 to 255');
+ const absoluteTask = path.resolve(taskFile);
+ const taskDir = path.dirname(absoluteTask);
+ const taskFileSha256 = taskSha256(fs.readFileSync(absoluteTask));
+ const manifest = readTaskManifest(absoluteTask);
+ const validation = validateTaskManifest(manifest, { taskFile: absoluteTask, verifyFiles: true, verifyTree: true });
+ if (!validation.valid) {
+ const error = new Error('task input validation failed: ' + validation.issues.map((item) => `${item.path}: ${item.message}`).join('; '));
+ error.kind = 'input-error';
+ throw error;
+ }
+ const loadedGraders = manifest.graders.map((reference) => {
+ const loaded = loadGraderSpec(reference.spec, taskDir);
+ if (!loaded.file || taskSha256(fs.readFileSync(loaded.file)) !== reference.sha256) {
+ const error = new Error(`grader ${reference.id} spec digest mismatch`);
+ error.kind = 'input-error';
+ throw error;
+ }
+ if (computeTreeSha256(loaded.root) !== reference.treeSha256) {
+ const error = new Error(`grader ${reference.id} tree digest mismatch`);
+ error.kind = 'input-error';
+ throw error;
+ }
+ const graderValidation = validateGraderSpec(loaded.spec);
+ if (!graderValidation.valid) {
+ const error = new Error(`grader ${reference.id} is invalid: ${graderValidation.errors.join('; ')}`);
+ error.kind = 'input-error';
+ throw error;
+ }
+ return { reference, ...loaded };
+ });
+ const trustedToolsCapture = captureTrustedTools();
+ const sourceRoot = resolveSourceRoot(manifest, { taskFile: absoluteTask });
+ const requestedOutputRoot = path.resolve(artifactRoot);
+ if (fs.existsSync(requestedOutputRoot) && fs.lstatSync(requestedOutputRoot).isSymbolicLink()) throw new Error('artifact root must not be a symlink');
+ const outputRoot = canonicalPath(requestedOutputRoot);
+ ensureOutsideSource(sourceRoot, outputRoot);
+ if (fs.existsSync(outputRoot) && fs.readdirSync(outputRoot).length) throw new Error('artifact root must not already contain files');
+ fs.mkdirSync(outputRoot, { recursive: true });
+ const sourceBefore = snapshotTree(sourceRoot, { errorOnSpecialFile: true, exclude: false });
+ const workspaceParent = fs.mkdtempSync(path.join(os.tmpdir(), 'sg-agent-task-'));
+ const workspaceRoot = path.join(workspaceParent, 'workspace');
+ try {
+ fs.cpSync(sourceRoot, workspaceRoot, { recursive: true, dereference: false, verbatimSymlinks: true });
+ const workspaceBeforeAgent = snapshotTree(workspaceRoot, { errorOnSpecialFile: true, exclude: false });
+ const patchPath = path.join(outputRoot, 'agent.patch');
+ const promptPath = path.join(outputRoot, 'TASK.md');
+ writeAtomic(promptPath, Buffer.from(manifest.instructions.text + '\n', 'utf8'));
+ const promptSha256 = sha256Bytes(fs.readFileSync(promptPath));
+ const values = { '{workspace}': workspaceRoot, '{patch}': patchPath, '{prompt}': promptPath, '{taskId}': manifest.taskId, '{artifacts}': outputRoot };
+ const argv = replaceTokens(agentArgv, values);
+ const agentRun = runCommand({ argv, cwd: workspaceRoot, allowedRoot: workspaceParent, timeoutMs: manifest.execution.timeoutMs, maxOutputBytes: manifest.execution.maxOutputBytes, env: {}, envAllowlist: ['PATH', 'HOME', 'TMPDIR', 'TMP', 'TEMP', 'LANG', 'LC_ALL', 'LC_CTYPE'] });
+ const agentAudit = agentResult(agentRun);
+ agentAudit.provider = agent.provider || null;
+ agentAudit.model = agent.model || null;
+ try { agentAudit.providerMetadata = agentRun.stdout.trim() ? JSON.parse(agentRun.stdout.trim().split('\n').pop()) : null; }
+ catch (_) { agentAudit.providerMetadata = null; }
+ let workspaceAfterAgent = null;
+ let sourceAfterAgent = null;
+ let verdict = 'passed';
+ let patchAudit = null;
+ const grades = [];
+ const secondaryErrors = [];
+ let termination = { kind: 'completed', processExitCode: 0, semanticVerdict: 'passed' };
+ const artifactList = [artifact(outputRoot, 'task-prompt', promptPath, 'text/markdown')];
+
+ try {
+ try {
+ workspaceAfterAgent = snapshotTree(workspaceRoot, { errorOnSpecialFile: true, exclude: false });
+ sourceAfterAgent = snapshotTree(sourceRoot, { errorOnSpecialFile: true, exclude: false });
+ } catch (snapshotError) {
+ const error = new Error('agent created an unsupported filesystem entry: ' + snapshotError.message);
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (!fs.existsSync(promptPath) || fs.lstatSync(promptPath).isSymbolicLink() || !fs.lstatSync(promptPath).isFile() || fs.lstatSync(promptPath).nlink > 1 || sha256Bytes(fs.readFileSync(promptPath)) !== promptSha256) {
+ const error = new Error('agent modified the immutable task prompt artifact');
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ const agentArtifactNames = fs.readdirSync(outputRoot).sort();
+ if (agentArtifactNames.some((name) => !['TASK.md', 'agent.patch'].includes(name))) {
+ const error = new Error('agent wrote undeclared files into the artifact root: ' + agentArtifactNames.filter((name) => !['TASK.md', 'agent.patch'].includes(name)).join(', '));
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (sourceAfterAgent.treeSha256 !== sourceBefore.treeSha256) {
+ const error = new Error('agent modified the source root outside the disposable workspace');
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (workspaceAfterAgent.treeSha256 !== workspaceBeforeAgent.treeSha256) {
+ const error = new Error('agent modified the workspace directly; it must emit only a patch');
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ const postAgentIntegrityIssues = [
+ ...integrityIssues({ absoluteTask, taskFileSha256, loadedGraders, externalBindings: integrityInputs }),
+ ...verifyTrustedTools(trustedToolsCapture),
+ ];
+ if (postAgentIntegrityIssues.length) {
+ const error = new Error('immutable task inputs changed during agent execution: ' + postAgentIntegrityIssues.join('; '));
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (agentRun.error || agentRun.exitCode !== 0) {
+ const error = new Error(agentRun.timedOut ? 'agent command timed out' : 'agent command failed');
+ error.kind = agentRun.timedOut || agentRun.error || new Set(agent.infraExitCodes || []).has(agentRun.exitCode) ? 'infra-error' : 'agent-failed';
+ throw error;
+ }
+ if (!fs.existsSync(patchPath)) {
+ const error = new Error('agent did not produce the required unified diff');
+ error.kind = 'agent-failed';
+ throw error;
+ }
+ if (fs.lstatSync(patchPath).isSymbolicLink() || !fs.lstatSync(patchPath).isFile() || fs.lstatSync(patchPath).nlink > 1) {
+ const error = new Error('agent.patch must be a regular non-symlink, non-hardlinked file');
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ const patchBytes = fs.readFileSync(patchPath);
+ artifactList.push(artifact(outputRoot, 'agent-patch', patchPath, 'text/x-diff'));
+ const applied = applyPatch({ manifest, workspaceRoot, patchBytes, taskId: manifest.taskId, timeoutMs: manifest.execution.timeoutMs });
+ patchAudit = applied.audit;
+ const patchAuditPath = path.join(outputRoot, 'patch-audit.json');
+ writeAtomic(patchAuditPath, Buffer.from(JSON.stringify(patchAudit, null, 2) + '\n'));
+ artifactList.push(artifact(outputRoot, 'patch-audit', patchAuditPath, 'application/json'));
+ const candidateTreeSha256 = 'sha256:' + applied.after.treeSha256;
+ let trustedTools;
+ let stagedGraders;
+ try {
+ trustedTools = stageTrustedTools(workspaceParent, trustedToolsCapture);
+ stagedGraders = stageGraders(loadedGraders, workspaceParent);
+ } catch (stagingError) {
+ const error = new Error('trusted grader staging failed: ' + stagingError.message);
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ for (const loadedGrader of stagedGraders) {
+ const { reference } = loadedGrader;
+ const preGradeToolIssues = verifyTrustedTools(trustedTools);
+ if (preGradeToolIssues.length) {
+ const error = new Error('trusted tool integrity failed before grading: ' + preGradeToolIssues.join('; '));
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (computeTreeSha256(loadedGrader.root) !== reference.treeSha256) {
+ const error = new Error(`staged grader ${reference.id} changed before invocation`);
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ const grade = gradeWorkspace({ spec: loadedGrader.spec, graderRoot: loadedGrader.root, workspaceRoot, artifactRoot: outputRoot, toolRoot: trustedTools.toolRoot, taskId: manifest.taskId, treeSha256: candidateTreeSha256, timeoutMs: manifest.execution.timeoutMs, maxOutputBytes: manifest.execution.maxOutputBytes });
+ grade.manifestWeight = reference.weight;
+ grade.gradeId = contentId('grade', grade, ['gradeId']);
+ grades.push(grade);
+ const gradeFile = path.join(outputRoot, `grade-${reference.id}.json`);
+ writeAtomic(gradeFile, Buffer.from(JSON.stringify(grade, null, 2) + '\n'));
+ artifactList.push(artifact(outputRoot, `grade-${reference.id}`, gradeFile, 'application/json'));
+ const postGradeToolIssues = verifyTrustedTools(trustedTools);
+ if (postGradeToolIssues.length) {
+ const error = new Error('trusted tool integrity failed during grading: ' + postGradeToolIssues.join('; '));
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (computeTreeSha256(loadedGrader.root) !== reference.treeSha256) {
+ const error = new Error(`staged grader ${reference.id} changed during invocation`);
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ }
+ const postGraderIntegrityIssues = integrityIssues({ absoluteTask, taskFileSha256, loadedGraders, externalBindings: integrityInputs });
+ if (postGraderIntegrityIssues.length) {
+ const error = new Error('immutable task inputs changed during grading: ' + postGraderIntegrityIssues.join('; '));
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ let workspaceAfterGraders;
+ try {
+ workspaceAfterGraders = snapshotTree(workspaceRoot, { errorOnSpecialFile: true, exclude: false });
+ } catch (snapshotError) {
+ const error = new Error('grader created an unsupported filesystem entry: ' + snapshotError.message);
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (workspaceAfterGraders.treeSha256 !== applied.after.treeSha256) {
+ const error = new Error('grader modified the immutable candidate workspace');
+ error.kind = 'policy-violation';
+ throw error;
+ }
+ if (!grades.length || grades.some((grade) => grade.verdict === 'failed')) {
+ verdict = 'grader-failed';
+ termination = { kind: 'completed', processExitCode: 1, semanticVerdict: verdict };
+ } else if (grades.some((grade) => grade.verdict === 'infra-error')) {
+ verdict = 'infra-error';
+ termination = { kind: 'infrastructure-error', processExitCode: 3, semanticVerdict: verdict, message: 'one or more grader checks had an infrastructure error' };
+ }
+ } catch (error) {
+ if (error.audit) {
+ patchAudit = error.audit;
+ const patchAuditPath = path.join(outputRoot, 'patch-audit.json');
+ writeAtomic(patchAuditPath, Buffer.from(JSON.stringify(patchAudit, null, 2) + '\n'));
+ artifactList.push(artifact(outputRoot, 'patch-audit', patchAuditPath, 'application/json'));
+ }
+ const candidateFailureObserved = grades.some((grade) => grade.verdict === 'failed');
+ if (candidateFailureObserved) {
+ secondaryErrors.push({ kind: error.kind || 'post-grade-error', message: error.message });
+ verdict = 'grader-failed';
+ termination = { kind: 'completed', processExitCode: 1, semanticVerdict: verdict, message: 'candidate failure observed; additional post-grade error: ' + error.message };
+ } else {
+ verdict = error.kind === 'policy-not-supported' ? 'patch-invalid' : error.kind || (error instanceof PatchError ? 'patch-invalid' : 'infra-error');
+ if (!['agent-failed', 'patch-invalid', 'policy-violation', 'input-error', 'infra-error'].includes(verdict)) verdict = 'patch-invalid';
+ termination = { kind: verdict === 'infra-error' ? 'infrastructure-error' : 'completed', processExitCode: verdict === 'infra-error' ? 3 : verdict === 'input-error' ? 2 : 1, semanticVerdict: verdict, message: error.message };
+ }
+ }
+
+ const unsupportedArtifacts = collectOutputArtifacts(outputRoot, artifactList);
+ if (unsupportedArtifacts.length && verdict !== 'policy-violation') {
+ const message = 'trusted grader output contains unsupported artifacts: ' + unsupportedArtifacts.join(', ');
+ if (grades.some((grade) => grade.verdict === 'failed') || verdict === 'grader-failed') {
+ secondaryErrors.push({ kind: 'unsupported-artifact', message });
+ verdict = 'grader-failed';
+ termination = { kind: 'completed', processExitCode: 1, semanticVerdict: verdict, message: 'candidate failure observed; additional post-grade error: ' + message };
+ } else {
+ verdict = 'policy-violation';
+ termination = { kind: 'completed', processExitCode: 1, semanticVerdict: verdict, message };
+ }
+ }
+ let sourceFinal = null;
+ let sourceFinalError = null;
+ try { sourceFinal = snapshotTree(sourceRoot, { errorOnSpecialFile: true, exclude: false }); }
+ catch (error) { sourceFinalError = error.message; }
+ const preserveCandidateFailure = (kind, message) => {
+ if (grades.some((grade) => grade.verdict === 'failed') || verdict === 'grader-failed') {
+ secondaryErrors.push({ kind, message });
+ verdict = 'grader-failed';
+ termination = { kind: 'completed', processExitCode: 1, semanticVerdict: verdict, message: 'candidate failure observed; additional final integrity error: ' + message };
+ } else {
+ verdict = 'policy-violation';
+ termination = { kind: 'completed', processExitCode: 1, semanticVerdict: verdict, message };
+ }
+ };
+ if (!sourceFinal) preserveCandidateFailure('source-snapshot', 'source root final snapshot failed: ' + sourceFinalError);
+ else if (sourceFinal.treeSha256 !== sourceBefore.treeSha256) preserveCandidateFailure('source-drift', 'source root changed during task execution');
+ const finalIntegrityIssues = integrityIssues({ absoluteTask, taskFileSha256, loadedGraders, externalBindings: integrityInputs });
+ if (finalIntegrityIssues.length) preserveCandidateFailure('immutable-input-drift', 'immutable task inputs changed during task execution: ' + finalIntegrityIssues.join('; '));
+ const report = buildRun({
+ runVersion: '1.0', runId: null,
+ task: {
+ taskId: manifest.taskId,
+ taskFileSha256,
+ sourceRevision: manifest.source.revision,
+ contractTreeSha256: manifest.source.treeSha256,
+ runnerSnapshotBeforeSha256: 'sha256:' + sourceBefore.treeSha256,
+ runnerSnapshotFinalSha256: sourceFinal ? 'sha256:' + sourceFinal.treeSha256 : null,
+ sourceUnchanged: Boolean(sourceFinal && sourceBefore.treeSha256 === sourceFinal.treeSha256),
+ sourceFinalError,
+ trialId,
+ },
+ agent: agentAudit,
+ patch: patchAudit,
+ grades,
+ secondaryErrors,
+ artifacts: artifactList,
+ verdict,
+ termination,
+ capabilities: { sourceCopy: true, disposableWorkspace: true, completeSecuritySnapshots: true, postAgentSourceTreeCheck: true, postPatchTreePolicy: true, postGraderTreeCheck: true, verifiedGraderStaging: true, verifiedToolStaging: true, candidateExtractionWorker: true, osSandbox: false, filesystemIsolation: false, networkIsolation: false, processIsolation: false, maliciousAgentGraderSecrecy: false },
+ });
+ const runFile = path.join(outputRoot, 'task-run.json');
+ writeAtomic(runFile, Buffer.from(JSON.stringify(report, null, 2) + '\n'));
+ if (keepWorkspace) report.workspaceRoot = workspaceRoot;
+ return { report, runFile, workspaceRoot: keepWorkspace ? workspaceRoot : null };
+ } finally {
+ if (!keepWorkspace) fs.rmSync(workspaceParent, { recursive: true, force: true });
+ }
+}
+
+module.exports = { runTask, loadGraderSpec, writeAtomic };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-tree-snapshot.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-tree-snapshot.js
new file mode 100644
index 0000000..b8ee46f
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-tree-snapshot.js
@@ -0,0 +1,231 @@
+'use strict';
+
+/* Deterministic, read-only filesystem tree snapshots and diffs. */
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { canonicalPath, matchesGlob } = require('./sg-path-policy.js');
+
+const DEFAULT_EXCLUDES = ['.git', '**/.git', '.git/**', '**/.git/**'];
+
+function compareNames(left, right) {
+ return left === right ? 0 : (left < right ? -1 : 1);
+}
+
+function sha256(value) {
+ return crypto.createHash('sha256').update(value).digest('hex');
+}
+
+function entryMode(stat) {
+ return stat.mode & 0o7777;
+}
+
+function normalizePatterns(value) {
+ if (value === false || value === null) return [];
+ if (value === undefined) return DEFAULT_EXCLUDES;
+ if (typeof value === 'string') return [value];
+ if (!Array.isArray(value) || value.some((pattern) => typeof pattern !== 'string')) {
+ throw new TypeError('exclude must be a string, an array of strings, false, or null');
+ }
+ return value;
+}
+
+function isExcluded(relativePath, patterns) {
+ return patterns.some((pattern) => matchesGlob(relativePath, pattern));
+}
+
+function snapshotEntry(absolutePath, relativePath, stat) {
+ const base = {
+ path: relativePath,
+ relativePath,
+ kind: null,
+ mode: entryMode(stat),
+ sha256: null,
+ size: 0,
+ target: null,
+ };
+ if (stat.isFile()) {
+ const bytes = fs.readFileSync(absolutePath);
+ return { ...base, kind: 'file', sha256: sha256(bytes), size: bytes.length };
+ }
+ if (stat.isDirectory()) return { ...base, kind: 'directory' };
+ if (stat.isSymbolicLink()) {
+ const target = fs.readlinkSync(absolutePath);
+ const bytes = Buffer.from(target);
+ return { ...base, kind: 'symlink', sha256: sha256(bytes), size: bytes.length, target };
+ }
+ return null;
+}
+
+function treeHash(entries) {
+ if (!Array.isArray(entries)) throw new TypeError('entries must be an array');
+ const normalized = entries.map((entry) => ({
+ ...entry,
+ path: entry.path === undefined ? entry.relativePath : entry.path,
+ }));
+ if (normalized.some((entry) => typeof entry.path !== 'string')) throw new TypeError('snapshot entry path must be a string');
+ const ordered = normalized.sort((left, right) => compareNames(left.path, right.path));
+ return sha256(Buffer.from(ordered.map((entry) => JSON.stringify([
+ entry.path,
+ entry.kind,
+ entry.mode,
+ entry.sha256,
+ entry.size,
+ entry.target,
+ ])).join('\n')));
+}
+
+function treeSha256(value, options = {}) {
+ return typeof value === 'string' ? snapshotTree(value, options).treeSha256 : treeHash(snapshotEntries(value));
+}
+
+function snapshotTree(root, options = {}) {
+ const physicalRoot = canonicalPath(root, { mustExist: true });
+ const rootStat = fs.lstatSync(physicalRoot);
+ if (!rootStat.isDirectory()) throw new TypeError(`snapshot root must be a directory: ${root}`);
+ const exclude = normalizePatterns(options.exclude);
+ const entries = [];
+
+ function visit(directory, prefix) {
+ const names = fs.readdirSync(directory).sort(compareNames);
+ for (const name of names) {
+ const relativePath = prefix ? `${prefix}/${name}` : name;
+ if (isExcluded(relativePath, exclude)) continue;
+ const absolutePath = path.join(directory, name);
+ const stat = fs.lstatSync(absolutePath);
+ const entry = snapshotEntry(absolutePath, relativePath, stat);
+ if (!entry) {
+ if (options.errorOnSpecialFile) throw new Error(`unsupported filesystem entry: ${relativePath}`);
+ continue;
+ }
+ entries.push(entry);
+ if (entry.kind === 'directory') visit(absolutePath, relativePath);
+ }
+ }
+
+ visit(physicalRoot, '');
+ return {
+ root: physicalRoot,
+ entries,
+ treeSha256: treeHash(entries),
+ };
+}
+
+function snapshotEntries(value) {
+ if (Array.isArray(value)) return value;
+ if (value && Array.isArray(value.entries)) return value.entries;
+ throw new TypeError('snapshot must be an entry array or an object with entries');
+}
+
+function indexEntries(snapshot) {
+ const out = new Map();
+ for (const entry of snapshotEntries(snapshot)) {
+ if (!entry || typeof (entry.path === undefined ? entry.relativePath : entry.path) !== 'string') throw new TypeError('snapshot entry path must be a string');
+ const relativePath = entry.path === undefined ? entry.relativePath : entry.path;
+ if (out.has(relativePath)) throw new Error(`duplicate snapshot entry: ${relativePath}`);
+ out.set(relativePath, entry);
+ }
+ return out;
+}
+
+function contentSignature(entry) {
+ return JSON.stringify([entry.kind, entry.sha256, entry.size, entry.target]);
+}
+
+function renameSignature(entry) {
+ if (entry.kind === 'directory') return null;
+ return JSON.stringify([entry.kind, entry.sha256, entry.size, entry.target]);
+}
+
+function diffTrees(before, after, options = {}) {
+ const beforeMap = indexEntries(before);
+ const afterMap = indexEntries(after);
+ let added = [...afterMap.keys()].filter((entryPath) => !beforeMap.has(entryPath)).sort();
+ let deleted = [...beforeMap.keys()].filter((entryPath) => !afterMap.has(entryPath)).sort();
+ const modified = [];
+ const modeChanged = [];
+ const typeChanged = [];
+ const renamed = [];
+
+ for (const entryPath of [...afterMap.keys()].filter((name) => beforeMap.has(name)).sort()) {
+ const oldEntry = beforeMap.get(entryPath);
+ const newEntry = afterMap.get(entryPath);
+ if (oldEntry.kind !== newEntry.kind) {
+ typeChanged.push({ path: entryPath, before: oldEntry, after: newEntry });
+ } else if (contentSignature(oldEntry) !== contentSignature(newEntry)) {
+ modified.push({ path: entryPath, before: oldEntry, after: newEntry, modeChanged: oldEntry.mode !== newEntry.mode });
+ } else if (oldEntry.mode !== newEntry.mode) {
+ modeChanged.push({ path: entryPath, before: oldEntry, after: newEntry });
+ }
+ }
+
+ if (options.detectRenames || options.renames) {
+ const available = new Map();
+ for (const entryPath of added) {
+ const signature = renameSignature(afterMap.get(entryPath));
+ if (signature === null) continue;
+ if (!available.has(signature)) available.set(signature, []);
+ available.get(signature).push(entryPath);
+ }
+ const renamedFrom = new Set();
+ const renamedTo = new Set();
+ for (const from of deleted) {
+ const oldEntry = beforeMap.get(from);
+ const signature = renameSignature(oldEntry);
+ const candidates = signature === null ? null : available.get(signature);
+ if (!candidates || candidates.length === 0) continue;
+ const to = candidates.shift();
+ const newEntry = afterMap.get(to);
+ renamed.push({
+ from,
+ to,
+ kind: newEntry.kind,
+ sha256: newEntry.sha256,
+ modeChanged: oldEntry.mode !== newEntry.mode,
+ before: oldEntry,
+ after: newEntry,
+ });
+ renamedFrom.add(from);
+ renamedTo.add(to);
+ }
+ deleted = deleted.filter((entryPath) => !renamedFrom.has(entryPath));
+ added = added.filter((entryPath) => !renamedTo.has(entryPath));
+ }
+
+ const addChanges = added.map((entryPath) => ({ change: 'add', path: entryPath, after: afterMap.get(entryPath) }));
+ const modifyChanges = modified.map((change) => ({ change: 'modify', ...change }));
+ const deleteChanges = deleted.map((entryPath) => ({ change: 'delete', path: entryPath, before: beforeMap.get(entryPath) }));
+ const modeChanges = modeChanged.map((change) => ({ change: 'mode', ...change }));
+ const typeChanges = typeChanged.map((change) => ({ change: 'type', ...change }));
+ const renameChanges = renamed.map((change) => ({ change: 'rename', path: change.to, ...change }));
+ const changes = [...addChanges, ...modifyChanges, ...deleteChanges, ...modeChanges, ...typeChanges, ...renameChanges]
+ .sort((left, right) => (left.path || left.from).localeCompare(right.path || right.from) || left.change.localeCompare(right.change));
+
+ return {
+ added: addChanges,
+ modified: modifyChanges,
+ deleted: deleteChanges,
+ modeChanged: modeChanges,
+ typeChanged: typeChanges,
+ renamed: renameChanges,
+ add: addChanges,
+ modify: modifyChanges,
+ delete: deleteChanges,
+ mode: modeChanges,
+ type: typeChanges,
+ renames: renameChanges,
+ changes,
+ changed: changes.length > 0,
+ };
+}
+
+module.exports = {
+ DEFAULT_EXCLUDES,
+ sha256,
+ treeHash,
+ treeSha256,
+ snapshotTree,
+ createTreeSnapshot: snapshotTree,
+ diffTrees,
+ diffTreeSnapshots: diffTrees,
+};
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/sg-visual-evidence.js b/research/agent-eval/results/executed-tooling/scripts/lib/sg-visual-evidence.js
new file mode 100644
index 0000000..987e181
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/sg-visual-evidence.js
@@ -0,0 +1,74 @@
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { contentId, isDigest, verifyArtifacts } = require('./sg-evidence-utils.js');
+
+function validateVisualEvidence(evidence, options = {}) {
+ const errors = [];
+ if (!evidence || typeof evidence !== 'object' || Array.isArray(evidence)) return { valid: false, errors: ['visual evidence must be an object'], status: 'not-assessed' };
+ if (evidence.evidenceVersion !== '1.0') errors.push('visual evidenceVersion must be 1.0');
+ if (evidence.kind !== 'visual-evidence') errors.push('visual kind must be visual-evidence');
+ for (const field of ['referenceTreeSha256', 'candidateTreeSha256', 'configurationSha256']) if (!isDigest(evidence[field])) errors.push(`visual ${field} must be sha256:`);
+ if (!evidence.producer || typeof evidence.producer.name !== 'string' || typeof evidence.producer.version !== 'string') errors.push('visual producer name/version are required');
+ if (!evidence.thresholds || typeof evidence.thresholds !== 'object') errors.push('visual thresholds are required');
+ else {
+ if (typeof evidence.thresholds.maxPixelDiffRatio !== 'number' || evidence.thresholds.maxPixelDiffRatio < 0 || evidence.thresholds.maxPixelDiffRatio > 1) errors.push('visual maxPixelDiffRatio must be between 0 and 1');
+ if (typeof evidence.thresholds.minComputedStyleScore !== 'number' || evidence.thresholds.minComputedStyleScore < 0 || evidence.thresholds.minComputedStyleScore > 1) errors.push('visual minComputedStyleScore must be between 0 and 1');
+ if (typeof evidence.thresholds.minCoverage !== 'number' || evidence.thresholds.minCoverage < 0 || evidence.thresholds.minCoverage > 1) errors.push('visual minCoverage must be between 0 and 1');
+ }
+ if (!Array.isArray(evidence.scenarios) || !evidence.scenarios.length) errors.push('visual scenarios must be a non-empty array');
+ if (!Array.isArray(evidence.artifacts) || !evidence.artifacts.length) errors.push('visual artifacts must be a non-empty array');
+ const artifactIds = verifyArtifacts(evidence.artifacts, options.baseDir, errors);
+ const artifactById = new Map((evidence.artifacts || []).filter((item) => item && typeof item.id === 'string').map((item) => [item.id, item]));
+ const scenarioIds = new Set();
+ const scenarioResults = [];
+ for (const [index, scenario] of (evidence.scenarios || []).entries()) {
+ const scenarioErrors = [];
+ if (!scenario || typeof scenario !== 'object' || typeof scenario.id !== 'string' || !scenario.id) scenarioErrors.push(`scenarios[${index}].id is required`);
+ else if (scenarioIds.has(scenario.id)) scenarioErrors.push(`scenarios[${index}].id is duplicated`);
+ else scenarioIds.add(scenario.id);
+ if (!scenario || !scenario.viewport || !Number.isInteger(scenario.viewport.width) || scenario.viewport.width <= 0 || !Number.isInteger(scenario.viewport.height) || scenario.viewport.height <= 0) scenarioErrors.push(`scenarios[${index}].viewport width/height must be positive integers`);
+ for (const field of ['pixelDiffRatio', 'computedStyleScore', 'coverage']) if (!scenario || typeof scenario[field] !== 'number' || !Number.isFinite(scenario[field]) || scenario[field] < 0 || scenario[field] > 1) scenarioErrors.push(`scenarios[${index}].${field} must be a finite number between 0 and 1`);
+ if (!scenario || !Number.isInteger(scenario.stabilityFailures) || scenario.stabilityFailures < 0) scenarioErrors.push(`scenarios[${index}].stabilityFailures must be a non-negative integer`);
+ const refs = [];
+ for (const field of ['referenceArtifact', 'candidateArtifact', 'diffArtifact']) {
+ const id = scenario && scenario[field];
+ const artifact = artifactById.get(id);
+ if (typeof id !== 'string' || !artifactIds.has(id)) scenarioErrors.push(`scenarios[${index}].${field} must reference a verified artifact`);
+ else if (typeof artifact.mediaType !== 'string' || !artifact.mediaType.startsWith('image/')) scenarioErrors.push(`scenarios[${index}].${field} must reference an image artifact`);
+ refs.push(id);
+ }
+ if (new Set(refs).size !== refs.length) scenarioErrors.push(`scenarios[${index}] reference, candidate, and diff artifacts must be distinct`);
+ const thresholdFailed = scenario && evidence.thresholds && (
+ scenario.pixelDiffRatio > evidence.thresholds.maxPixelDiffRatio
+ || scenario.computedStyleScore < evidence.thresholds.minComputedStyleScore
+ || scenario.coverage < evidence.thresholds.minCoverage
+ || Number(scenario.stabilityFailures || 0) > 0
+ );
+ errors.push(...scenarioErrors);
+ scenarioResults.push({ id: scenario && scenario.id || `scenario-${index + 1}`, status: scenarioErrors.length ? 'not-assessed' : thresholdFailed ? 'failed' : 'passed', errors: scenarioErrors });
+ }
+ const expectedId = contentId('visual', evidence, ['evidenceId']);
+ if (evidence.evidenceId !== expectedId) errors.push(`visual evidenceId mismatch (expected ${expectedId})`);
+ const complete = errors.length === 0 && scenarioResults.length > 0 && scenarioResults.every((item) => item.status !== 'not-assessed');
+ return {
+ valid: errors.length === 0,
+ errors,
+ status: !complete ? 'not-assessed' : scenarioResults.some((item) => item.status === 'failed') ? 'failed' : 'passed',
+ scenarios: scenarioResults,
+ summary: {
+ total: scenarioResults.length,
+ passed: scenarioResults.filter((item) => item.status === 'passed').length,
+ failed: scenarioResults.filter((item) => item.status === 'failed').length,
+ notAssessed: scenarioResults.filter((item) => item.status === 'not-assessed').length,
+ },
+ };
+}
+
+function readVisualEvidence(file) {
+ const evidence = JSON.parse(fs.readFileSync(file, 'utf8'));
+ const validation = validateVisualEvidence(evidence, { baseDir: path.dirname(path.resolve(file)) });
+ return { evidence, validation };
+}
+
+module.exports = { validateVisualEvidence, readVisualEvidence };
diff --git a/research/agent-eval/results/executed-tooling/scripts/lib/task-run.schema.json b/research/agent-eval/results/executed-tooling/scripts/lib/task-run.schema.json
new file mode 100644
index 0000000..22cde09
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/lib/task-run.schema.json
@@ -0,0 +1,21 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://sg.local/contracts/task-run-v1.json",
+ "title": "SG Agent TaskRun v1.0",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["runVersion", "runId", "task", "agent", "patch", "grades", "secondaryErrors", "artifacts", "verdict", "termination", "capabilities"],
+ "properties": {
+ "runVersion": { "const": "1.0" },
+ "runId": { "type": "string", "pattern": "^task-run:[0-9a-f]{64}$" },
+ "task": { "type": "object" },
+ "agent": { "type": "object" },
+ "patch": { "type": ["object", "null"] },
+ "grades": { "type": "array" },
+ "secondaryErrors": { "type": "array" },
+ "artifacts": { "type": "array" },
+ "verdict": { "enum": ["passed", "agent-failed", "patch-invalid", "policy-violation", "grader-failed", "input-error", "infra-error"] },
+ "termination": { "type": "object" },
+ "capabilities": { "type": "object" }
+ }
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-data-pack b/research/agent-eval/results/executed-tooling/scripts/sg-data-pack
new file mode 100755
index 0000000..da41a74
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-data-pack
@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+/*
+ * sg-data-pack — component-library data-layer normalization CLI
+ *
+ * Usage:
+ * sg-data-pack extract [--check] Extract embedded data into a Data Pack, then validate + equivalence-test
+ * sg-data-pack compile [--domain-schema fragment.json] [--check] Generate reviewed data.js/schema without erasing custom domain constraints
+ * sg-data-pack validate [--strict] [--verify-hash] Standalone Data Pack validation
+ * sg-data-pack rules [--strict] Execute library-level data rules
+ * sg-data-pack diff [--json] Structural diff between two Data Packs
+ * sg-data-pack templatize [--out dir] [--prefix n] Derive item template from repeated HTML instances
+ * sg-data-pack alias-candidates [--json] [--top n] Rank alias candidates for crawled names
+ * sg-data-pack types [--out file.d.ts] [--name N] Generate TypeScript declarations from a pack
+ * sg-data-pack recrawl-skeleton [--out dir] Recrawl normalization pipeline (alias hits/misses + cross-check + review report)
+ * sg-data-pack data-surface-import [--out report.json] [--allow-review-required] Read a ui-dismantler Data Surface Manifest without generating a Data Pack
+ * sg-data-pack candidate --records records.json --out candidate.json Apply explicit Review Decisions
+ * sg-data-pack report [options] Generate a unified Library Evolution Report
+ * sg-data-pack task Validate or execute an Agent Task Manifest
+ * sg-data-pack grade Execute a task-specific grader
+ * sg-data-pack evidence Validate runtime or visual evidence
+ * sg-data-pack experiment --out Run a success-rate experiment
+ * sg-data-pack schema Print the contract-schema path
+ * sg-data-pack loader Print the runtime-validator path (copy into a library's lib/src/)
+ */
+'use strict';
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const [, , cmd, ...rest] = process.argv;
+const here = __dirname;
+
+function run(script, args) {
+ const r = spawnSync(process.execPath, [path.join(here, script), ...args], { stdio: 'inherit' });
+ process.exit(r.status === null ? 1 : r.status);
+}
+
+function printHelp() {
+ console.log(`sg-data-pack — component-library data-layer normalization CLI
+
+Usage:
+ sg-data-pack extract [--check] [--compare-existing] [--force] Bootstrap from source literals + equivalence-test
+ sg-data-pack compile [--domain-schema fragment.json] [--check] Generate data.js/schema and preserve explicit domain constraints
+ sg-data-pack validate [--strict] [--verify-hash] [--asset-root dir] Validate a Data Pack
+ sg-data-pack rules [--strict] Execute library-level rules
+ sg-data-pack diff [--json] Structural diff + derivation impact
+ sg-data-pack templatize [--out dir] Derive a byte-exact repeated-item template
+ sg-data-pack alias-candidates [--json] Rank crawl-name alias candidates
+ sg-data-pack types [--out file] [--name N] Generate TypeScript declarations
+ sg-data-pack recrawl-skeleton Build a crawl review report
+ sg-data-pack data-surface-import [--out report.json] [--allow-review-required] Read a ui-dismantler Data Surface Manifest
+ sg-data-pack candidate --records --out Apply explicit Review Decisions
+ sg-data-pack report [--config config.js] [--baseline old.json] [--review review-report.json] [--audit candidate-audit.json] [--strict] [--verify-hash] [--out dir] [--json]
+ sg-data-pack task Validate or execute an Agent Task Manifest
+ sg-data-pack grade [--artifacts dir] [--tree sha256:...] [--json]
+ sg-data-pack evidence [--json]
+ sg-data-pack experiment --out [--json]
+ sg-data-pack schema Print the contract-schema path
+ sg-data-pack loader Print the runtime-validator path`);
+}
+
+switch (cmd) {
+ case 'extract':
+ run('sg-pack-extract.js', rest);
+ break;
+ case 'compile':
+ run('sg-pack-compile.js', rest);
+ break;
+ case 'validate':
+ run('sg-pack-validate.js', rest);
+ break;
+ case 'rules':
+ run('sg-pack-rules.js', rest);
+ break;
+ case 'diff':
+ run('sg-pack-diff.js', rest);
+ break;
+ case 'templatize':
+ run('sg-pack-templatize.js', rest);
+ break;
+ case 'alias-candidates':
+ run('sg-pack-alias-candidates.js', rest);
+ break;
+ case 'types':
+ run('sg-pack-types.js', rest);
+ break;
+ case 'recrawl-skeleton':
+ run('sg-pack-recrawl-skeleton.js', rest);
+ break;
+ case 'data-surface-import':
+ run('sg-pack-data-surface.js', rest);
+ break;
+ case 'candidate':
+ run('sg-pack-candidate.js', rest);
+ break;
+ case 'report':
+ run('sg-pack-report.js', rest);
+ break;
+ case 'task':
+ run('sg-pack-task.js', rest);
+ break;
+ case 'grade':
+ run('sg-pack-grade.js', rest);
+ break;
+ case 'evidence':
+ run('sg-pack-evidence.js', rest);
+ break;
+ case 'experiment':
+ run('sg-pack-experiment.js', rest);
+ break;
+ case 'schema':
+ console.log(path.join(here, 'lib', 'data-pack.schema.json'));
+ break;
+ case 'loader':
+ console.log(path.join(here, 'lib', 'sg-data-loader.js'));
+ break;
+ case 'help':
+ case '--help':
+ case '-h':
+ printHelp();
+ break;
+ default:
+ printHelp();
+ process.exit(cmd ? 2 : 0);
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-alias-candidates.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-alias-candidates.js
new file mode 100644
index 0000000..d38c274
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-alias-candidates.js
@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-alias-candidates.js — alias candidate generator for crawl normalization.
+ *
+ * Input: a Data Pack + a list of crawled names (JSON array or newline text).
+ * For each name: report HIT (resolves to a stable id via aliases) or MISS with
+ * candidate ids ranked by similarity (normalized Levenshtein over entity ids,
+ * entity display names per kindNameFields/name, and existing alias keys).
+ *
+ * Usage:
+ * sg-data-pack alias-candidates [--json] [--top ]
+ *
+ * Exit: 0 = all names hit, 1 = misses found (review candidates and extend aliases),
+ * 2 = usage/io error.
+ *
+ * Workflow: run after a crawl, before writing any reference. Extend the pack's
+ * aliases with confirmed mappings only — candidates are suggestions, not truth.
+ */
+'use strict';
+const fs = require('fs');
+
+const [, , packPath, namesPath, ...rest] = process.argv;
+if (!packPath || !namesPath) {
+ console.error('Usage: sg-data-pack alias-candidates [--json] [--top ]');
+ process.exit(2);
+}
+const asJson = rest.includes('--json');
+const topN = (() => { const i = rest.indexOf('--top'); return i >= 0 ? parseInt(rest[i + 1], 10) : 3; })();
+
+function load(p) {
+ try { return fs.readFileSync(p, 'utf8'); } catch (e) { console.error(`Cannot read ${p}: ${e.message}`); process.exit(2); }
+}
+const pack = JSON.parse(load(packPath));
+const raw = load(namesPath).trim();
+const names = raw.startsWith('[') ? JSON.parse(raw) : raw.split('\n').map((s) => s.trim()).filter(Boolean);
+
+/* ---------- similarity ---------- */
+function normalize(s) {
+ return String(s).toLowerCase()
+ .normalize('NFD').replace(/[̀-ͯ]/g, '') // strip accents (Rosé -> rose)
+ .replace(/[\s·・\-_.'"()()]+/g, ''); // strip separators
+}
+function lev(a, b) {
+ const m = a.length, n = b.length;
+ if (!m) return n; if (!n) return m;
+ const dp = new Uint16Array(n + 1);
+ for (let j = 0; j <= n; j++) dp[j] = j;
+ for (let i = 1; i <= m; i++) {
+ let prev = dp[0]; dp[0] = i;
+ for (let j = 1; j <= n; j++) {
+ const t = dp[j];
+ dp[j] = Math.min(dp[j] + 1, dp[j - 1] + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1));
+ prev = t;
+ }
+ }
+ return dp[n];
+}
+function pairScore(a, b) {
+ const na = normalize(a), nb = normalize(b);
+ if (!na || !nb) return 0;
+ if (na === nb) return 1;
+ // prefix match (Roseanne ⊃ Rosé, 程兵 ⊃ 程) — strong signal for name variants
+ if (na.startsWith(nb) || nb.startsWith(na)) {
+ return 0.6 + 0.35 * (Math.min(na.length, nb.length) / Math.max(na.length, nb.length));
+ }
+ if (na.includes(nb) || nb.includes(na)) return Math.min(na.length, nb.length) / Math.max(na.length, nb.length) * 0.95;
+ return 1 - lev(na, nb) / Math.max(na.length, nb.length);
+}
+function similarity(a, b) {
+ const full = pairScore(a, b);
+ // token-level: multi-word names match on their strongest token pair
+ const ta = String(a).split(/[\s·・\-_]+/).filter(Boolean);
+ const tb = String(b).split(/[\s·・\-_]+/).filter(Boolean);
+ let best = 0;
+ for (const x of ta) for (const y of tb) best = Math.max(best, pairScore(x, y));
+ return Math.max(full, best * 0.9);
+}
+
+/* ---------- candidate surface ---------- */
+const nameField = (id, e) => {
+ const knf = (pack.kindNameFields || {})[e.kind];
+ return (knf && e[knf]) || e.name || id;
+};
+const surface = []; // [{id, text, via}]
+for (const [id, e] of Object.entries(pack.entities || {})) {
+ surface.push({ id, text: id, via: 'id' });
+ const dn = nameField(id, e);
+ if (dn && dn !== id) surface.push({ id, text: dn, via: 'name' });
+}
+for (const [alias, target] of Object.entries(pack.aliases || {})) {
+ const id = target && typeof target === 'object' ? target.id : target;
+ if (id && (pack.entities || {})[id]) surface.push({ id, text: alias, via: 'alias' });
+}
+
+const resolve = (name) => {
+ if ((pack.entities || {})[name]) return name;
+ let via = (pack.aliases || {})[name];
+ if (via && typeof via === 'object') via = via.id;
+ return via && (pack.entities || {})[via] ? via : null;
+};
+
+/* ---------- run ---------- */
+const results = [];
+let misses = 0;
+for (const name of names) {
+ const hit = resolve(name);
+ if (hit) { results.push({ name, status: 'hit', id: hit }); continue; }
+ misses++;
+ const scored = new Map(); // id -> best {score, via, text}
+ for (const s of surface) {
+ const sc = similarity(name, s.text);
+ const cur = scored.get(s.id);
+ if (!cur || sc > cur.score) scored.set(s.id, { score: sc, via: s.via, text: s.text });
+ }
+ const candidates = [...scored.entries()]
+ .map(([id, v]) => ({ id, score: +v.score.toFixed(3), matchedOn: `${v.via}:"${v.text}"` }))
+ .filter((c) => c.score >= 0.5)
+ .sort((a, b) => b.score - a.score)
+ .slice(0, topN);
+ results.push({ name, status: 'miss', candidates });
+}
+
+if (asJson) {
+ console.log(JSON.stringify(results, null, 2));
+} else {
+ for (const r of results) {
+ if (r.status === 'hit') console.log(`HIT ${r.name} -> ${r.id}`);
+ else {
+ console.log(`MISS ${r.name}`);
+ for (const c of r.candidates) console.log(` ${c.score} ${c.id} (${c.matchedOn})`);
+ if (!r.candidates.length) console.log(' (no candidates above 0.50 — likely a genuinely new entity)');
+ }
+ }
+ console.log(`\n${names.length} names: ${names.length - misses} hit, ${misses} miss`);
+}
+process.exit(misses ? 1 : 0);
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-candidate.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-candidate.js
new file mode 100644
index 0000000..07b9d72
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-candidate.js
@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-candidate.js — apply explicit Review Decisions to a Data Pack.
+ *
+ * Usage:
+ * sg-data-pack candidate
+ * --out --records [--audit ] [--strict] [--force]
+ *
+ * Exit code: 0 = candidate written; 1 = review/data gate failed; 2 = usage/input error.
+ */
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const os = require('node:os');
+const crypto = require('node:crypto');
+const { buildCandidate, CandidateError } = require('./lib/sg-pack-candidate.js');
+
+const [, , baselinePath, reportPath, decisionsPath, ...flags] = process.argv;
+if (!baselinePath || !reportPath || !decisionsPath) {
+ console.error('Usage: sg-data-pack candidate --out --records [--audit ] [--strict] [--force]');
+ process.exit(2);
+}
+
+function flag(name) {
+ const index = flags.indexOf(name);
+ if (index < 0) return null;
+ if (index + 1 >= flags.length || flags[index + 1].startsWith('--')) {
+ console.error(`${name} requires a value`);
+ process.exit(2);
+ }
+ return flags[index + 1];
+}
+
+const outPath = flag('--out');
+const auditPath = flag('--audit') || (outPath ? `${outPath}.audit.json` : null);
+const recordsPath = flag('--records');
+const strict = flags.includes('--strict');
+const force = flags.includes('--force');
+if (!outPath) {
+ console.error('candidate requires --out ');
+ process.exit(2);
+}
+if (!recordsPath) {
+ console.error('candidate requires --records to bind the review report to its crawl input');
+ process.exit(2);
+}
+
+function resolve(file) { return path.resolve(file); }
+function canonicalPath(file) {
+ let absolute = resolve(file);
+ const missing = [];
+ while (!fs.existsSync(absolute)) {
+ const parent = path.dirname(absolute);
+ if (parent === absolute) break;
+ missing.unshift(path.basename(absolute));
+ absolute = parent;
+ }
+ try { absolute = fs.realpathSync(absolute); } catch (_) { /* checked below by input reads */ }
+ return path.join(absolute, ...missing);
+}
+function sameFile(left, right) {
+ const canonicalLeft = canonicalPath(left);
+ const canonicalRight = canonicalPath(right);
+ const caseInsensitiveFs = process.platform === 'darwin' || process.platform === 'win32';
+ if (canonicalLeft === canonicalRight || (caseInsensitiveFs && canonicalLeft.toLowerCase() === canonicalRight.toLowerCase())) return true;
+ try {
+ const a = fs.statSync(left);
+ const b = fs.statSync(right);
+ return a.dev === b.dev && a.ino === b.ino;
+ } catch (_) {
+ return false;
+ }
+}
+const inputPaths = [baselinePath, reportPath, decisionsPath, recordsPath].filter(Boolean).map(resolve);
+const outputPaths = [outPath, auditPath].map(resolve);
+if (sameFile(outputPaths[0], outputPaths[1])) {
+ console.error('candidate and audit output paths must be different');
+ process.exit(2);
+}
+if (outputPaths.some((output) => inputPaths.some((input) => sameFile(output, input)))) {
+ console.error('candidate output paths must not overwrite an input file');
+ process.exit(2);
+}
+
+function acquireOutputLock(paths) {
+ const caseInsensitiveFs = process.platform === 'darwin' || process.platform === 'win32';
+ const lockPaths = [...new Set(paths.map((file) => {
+ const canonical = canonicalPath(file);
+ return caseInsensitiveFs ? canonical.toLowerCase() : canonical;
+ }))]
+ .sort()
+ .map((target) => path.join(
+ os.tmpdir(),
+ 'sg-data-pack-candidate-' + crypto.createHash('sha256').update(target).digest('hex') + '.lock',
+ ));
+ const acquired = [];
+ const release = () => {
+ for (const lockPath of acquired.splice(0)) {
+ try { fs.unlinkSync(lockPath); } catch (_) { /* already released */ }
+ }
+ };
+ try {
+ for (const lockPath of lockPaths) {
+ let acquiredOne = false;
+ for (let attempt = 0; attempt < 2 && !acquiredOne; attempt += 1) {
+ try {
+ let fd;
+ try {
+ fd = fs.openSync(lockPath, 'wx', 0o600);
+ fs.writeFileSync(fd, String(process.pid));
+ } finally {
+ if (fd !== undefined) fs.closeSync(fd);
+ }
+ acquired.push(lockPath);
+ acquiredOne = true;
+ } catch (error) {
+ if (error.code !== 'EEXIST') throw error;
+ let owner = null;
+ try { owner = Number(fs.readFileSync(lockPath, 'utf8')); } catch (_) { /* raced with cleanup */ }
+ if (!owner) throw new Error('candidate output is locked by another process');
+ try {
+ process.kill(owner, 0);
+ throw new Error('candidate output is locked by process ' + owner);
+ } catch (probeError) {
+ if (probeError.message.indexOf('locked by process') !== -1 || probeError.code !== 'ESRCH') throw probeError;
+ try { fs.unlinkSync(lockPath); } catch (_) { throw new Error('candidate output is locked by another process'); }
+ }
+ }
+ }
+ if (!acquiredOne) throw new Error('candidate output lock could not be acquired');
+ }
+ let released = false;
+ return () => { if (!released) { released = true; release(); } };
+ } catch (error) {
+ release();
+ throw error;
+ }
+}
+let releaseOutputLock;
+try { releaseOutputLock = acquireOutputLock(outputPaths); }
+catch (error) { console.error('candidate: ' + error.message); process.exit(2); }
+process.on('exit', () => { if (releaseOutputLock) releaseOutputLock(); });
+
+if (!force && outputPaths.some((file) => fs.existsSync(file))) {
+ console.error('candidate output already exists; use --force to replace it');
+ process.exit(2);
+}
+
+function readBytes(file, label) {
+ try { return fs.readFileSync(file); }
+ catch (error) { throw new CandidateError(`${label}: ${error.message}`, 2); }
+}
+function parse(bytes, label) {
+ try { return JSON.parse(bytes.toString('utf8')); }
+ catch (error) { throw new CandidateError(`${label}: ${error.message}`, 2); }
+}
+
+let result;
+try {
+ const baselineBytes = readBytes(baselinePath, 'Cannot read baseline Data Pack');
+ const reportBytes = readBytes(reportPath, 'Cannot read review report');
+ const decisionsBytes = readBytes(decisionsPath, 'Cannot read review decisions');
+ const recordsBytes = recordsPath ? readBytes(recordsPath, 'Cannot read records') : undefined;
+ const baselinePack = parse(baselineBytes, 'Invalid baseline Data Pack');
+ const report = parse(reportBytes, 'Invalid review report');
+ const decisions = parse(decisionsBytes, 'Invalid review decisions');
+ const rawRecords = parse(recordsBytes, 'Invalid records');
+ if (!report.candidateReview || !report.candidateReview.reportId) {
+ throw new CandidateError('review report is not candidate-ready; rerun recrawl-skeleton with --candidate-ready', 2);
+ }
+ result = buildCandidate({
+ baselinePack,
+ baselineBytes,
+ report,
+ decisions,
+ decisionsBytes,
+ strict,
+ recordsBytes,
+ rawRecords,
+ });
+} catch (error) {
+ const exitCode = error instanceof CandidateError && error.exitCode ? error.exitCode : 2;
+ console.error(`candidate: ${error.message}`);
+ process.exit(exitCode);
+}
+
+function atomicWritePair(firstPath, firstBytes, secondPath, secondBytes) {
+ const tempPaths = [firstPath, secondPath].map((file) => path.join(
+ path.dirname(file),
+ `.${path.basename(file)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`,
+ ));
+ const backups = [];
+ const installed = [];
+ const writeTemp = (file, bytes) => {
+ fs.mkdirSync(path.dirname(file), { recursive: true });
+ let fd;
+ try {
+ fd = fs.openSync(file, 'wx', 0o600);
+ fs.writeFileSync(fd, bytes);
+ fs.fsyncSync(fd);
+ } finally {
+ if (fd !== undefined) fs.closeSync(fd);
+ }
+ };
+ const cleanupTemps = () => tempPaths.forEach((file) => { try { fs.unlinkSync(file); } catch (_) { /* absent */ } });
+ const cleanupInstalled = () => installed.forEach((file) => { try { fs.unlinkSync(file); } catch (_) { /* absent */ } });
+ try {
+ writeTemp(tempPaths[0], firstBytes);
+ writeTemp(tempPaths[1], secondBytes);
+ for (const file of [firstPath, secondPath]) {
+ if (!fs.existsSync(file)) continue;
+ const backup = `${file}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.bak`;
+ fs.renameSync(file, backup);
+ backups.push({ file, backup });
+ }
+ fs.renameSync(tempPaths[0], firstPath);
+ installed.push(firstPath);
+ fs.renameSync(tempPaths[1], secondPath);
+ installed.push(secondPath);
+ for (const { backup } of backups) { try { fs.unlinkSync(backup); } catch (_) { /* best effort cleanup */ } }
+ } catch (error) {
+ cleanupTemps();
+ cleanupInstalled();
+ const rollbackErrors = [];
+ for (const { file, backup } of backups) {
+ try { if (fs.existsSync(file)) fs.unlinkSync(file); } catch (rollbackError) { rollbackErrors.push(`${file}: ${rollbackError.message}`); }
+ try { if (fs.existsSync(backup)) fs.renameSync(backup, file); } catch (rollbackError) { rollbackErrors.push(`${file}: ${rollbackError.message}`); }
+ }
+ if (rollbackErrors.length) throw new Error(`${error.message}; rollback incomplete: ${rollbackErrors.join('; ')}`);
+ throw error;
+ }
+}
+
+try {
+ atomicWritePair(outPath, result.candidateBytes, auditPath, result.auditBytes);
+} catch (error) {
+ console.error('candidate: could not write outputs: ' + error.message);
+ process.exit(2);
+}
+
+console.log(`candidate: wrote ${outPath}`);
+console.log(`audit: wrote ${auditPath}`);
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-compile.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-compile.js
new file mode 100644
index 0000000..51e3811
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-compile.js
@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { sameFile } = require('./lib/sg-path-policy.js');
+require('./lib/sg-data-loader.js');
+
+function usage() {
+ console.error('Usage: sg-data-pack compile [--domain-schema ] [--check]');
+ process.exit(2);
+}
+
+const argv = process.argv.slice(2);
+const fileArg = argv.shift();
+if (!fileArg || fileArg.startsWith('--')) usage();
+let checkOnly = false;
+let domainSchemaArg = null;
+while (argv.length) {
+ const argument = argv.shift();
+ if (argument === '--check' && !checkOnly) checkOnly = true;
+ else if (argument === '--domain-schema' && domainSchemaArg === null && argv.length && !argv[0].startsWith('--')) domainSchemaArg = argv.shift();
+ else usage();
+}
+
+function canonical(value) {
+ if (Array.isArray(value)) return value.map(canonical);
+ if (!value || typeof value !== 'object') return value;
+ return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])]));
+}
+function equivalent(left, right) { return JSON.stringify(canonical(left)) === JSON.stringify(canonical(right)); }
+function readRegularJson(file, label) {
+ const stat = fs.lstatSync(file);
+ if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`${label} must be a regular non-symlink file`);
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+const file = path.resolve(fileArg);
+const directory = path.dirname(file);
+const dataJs = path.join(directory, 'data.js');
+const schemaFile = path.join(directory, 'data.schema.json');
+const domainSchemaFile = domainSchemaArg === null ? null : path.resolve(domainSchemaArg);
+if (sameFile(file, dataJs) || sameFile(file, schemaFile) || sameFile(dataJs, schemaFile)
+ || domainSchemaFile && [file, dataJs, schemaFile].some((candidate) => sameFile(domainSchemaFile, candidate))) {
+ console.error('compile input and output paths must be distinct');
+ process.exit(2);
+}
+try {
+ const pack = readRegularJson(file, 'data.json');
+ const validation = globalThis.SGDataLoader.validate(pack);
+ if (validation.errors.length) throw new Error('Data Pack is invalid: ' + validation.errors.join('; '));
+ const schema = JSON.parse(fs.readFileSync(path.join(__dirname, 'lib', 'data-pack.schema.json'), 'utf8'));
+ const baseDomainSchema = schema.properties.domain;
+ if (domainSchemaFile) {
+ const fragment = readRegularJson(domainSchemaFile, 'domain schema fragment');
+ if (typeof fragment !== 'boolean' && (!fragment || typeof fragment !== 'object' || Array.isArray(fragment))) throw new Error('domain schema fragment must be a JSON object or boolean schema');
+ schema.properties.domain = fragment;
+ } else if (fs.existsSync(schemaFile)) {
+ const existing = readRegularJson(schemaFile, 'existing data.schema.json');
+ const hasExistingDomain = existing && existing.properties && Object.prototype.hasOwnProperty.call(existing.properties, 'domain');
+ const existingDomain = hasExistingDomain ? existing.properties.domain : undefined;
+ if (hasExistingDomain && !equivalent(existingDomain, baseDomainSchema)) {
+ throw new Error(`existing data.schema.json contains a custom domain schema; rerun with --domain-schema to preserve it (${schemaFile})`);
+ }
+ }
+ schema.$id = `https://sg.local/${pack.meta.id}/data.schema.json`;
+ schema.title = `SG Data Pack ${pack.schemaVersion} — ${pack.meta.id} data contract`;
+ const outputs = [
+ { file: dataJs, bytes: Buffer.from('// @generated from reviewed data.json by sg-data-pack compile\n' + 'globalThis.SG_DATA_PACK = ' + JSON.stringify(pack) + ';\n') },
+ { file: schemaFile, bytes: Buffer.from(JSON.stringify(schema, null, 2) + '\n') },
+ ];
+ for (const output of outputs) if (fs.existsSync(output.file) && fs.lstatSync(output.file).isSymbolicLink()) throw new Error('compile output must not be a symlink: ' + output.file);
+ if (checkOnly) {
+ const drift = outputs.filter((output) => !fs.existsSync(output.file) || !fs.readFileSync(output.file).equals(output.bytes));
+ if (drift.length) {
+ drift.forEach((output) => console.error('compile output drift: ' + output.file));
+ process.exit(1);
+ }
+ console.log('✔ Compiled outputs match reviewed data.json');
+ process.exit(0);
+ }
+ const staged = outputs.map((output) => ({ ...output, temp: path.join(directory, `.${path.basename(output.file)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`), backup: null }));
+ try {
+ for (const output of staged) { fs.writeFileSync(output.temp, output.bytes, { flag: 'wx', mode: 0o600 }); }
+ for (const output of staged) if (fs.existsSync(output.file)) { output.backup = `${output.file}.${process.pid}.bak`; fs.renameSync(output.file, output.backup); }
+ for (const output of staged) fs.renameSync(output.temp, output.file);
+ for (const output of staged) if (output.backup) fs.unlinkSync(output.backup);
+ } catch (error) {
+ for (const output of staged) { try { fs.rmSync(output.temp, { force: true }); } catch (_) { /* absent */ } }
+ for (const output of staged) {
+ try { if (output.backup) { fs.rmSync(output.file, { force: true }); fs.renameSync(output.backup, output.file); } } catch (_) { /* reported below */ }
+ }
+ throw error;
+ }
+ outputs.forEach((output) => console.log('compiled: ' + output.file));
+} catch (error) {
+ console.error('compile: ' + error.message);
+ process.exit(error instanceof SyntaxError || error.code === 'ENOENT' ? 2 : 1);
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-data-surface.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-data-surface.js
new file mode 100755
index 0000000..64d1c75
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-data-surface.js
@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+'use strict';
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
+const { readDataSurfaceManifest } = require('./lib/sg-data-surface-import.js');
+
+function usage() {
+ return 'Usage: sg-data-pack data-surface-import [--out report.json] [--allow-review-required]';
+}
+
+function parseArgs(argv) {
+ let file = null;
+ let out = null;
+ let allowReviewRequired = false;
+ const seen = new Set();
+ for (let i = 0; i < argv.length; i += 1) {
+ const arg = argv[i];
+ if (arg === '--out') {
+ if (seen.has(arg)) throw new Error('duplicate option: --out');
+ seen.add(arg);
+ if (i + 1 >= argv.length || argv[i + 1].startsWith('--')) throw new Error('--out requires a value');
+ out = argv[++i];
+ continue;
+ }
+ if (arg === '--allow-review-required') {
+ if (seen.has(arg)) throw new Error('duplicate option: --allow-review-required');
+ seen.add(arg);
+ allowReviewRequired = true;
+ continue;
+ }
+ if (arg.startsWith('-')) throw new Error('unknown option: ' + arg);
+ if (file !== null) throw new Error('unexpected positional argument: ' + arg);
+ file = arg;
+ }
+ if (!file) throw new Error('manifest path is required');
+ return { file, out, allowReviewRequired };
+}
+
+function canonicalPath(file) {
+ let absolute = path.resolve(file);
+ const missing = [];
+ while (!fs.existsSync(absolute)) {
+ const parent = path.dirname(absolute);
+ if (parent === absolute) break;
+ missing.unshift(path.basename(absolute));
+ absolute = parent;
+ }
+ try { absolute = fs.realpathSync(absolute); } catch (_) { /* input reads report the concrete error */ }
+ return path.join(absolute, ...missing);
+}
+
+function sameFile(left, right) {
+ const canonicalLeft = canonicalPath(left);
+ const canonicalRight = canonicalPath(right);
+ const caseInsensitiveFs = process.platform === 'darwin' || process.platform === 'win32';
+ if (canonicalLeft === canonicalRight || (caseInsensitiveFs && canonicalLeft.toLowerCase() === canonicalRight.toLowerCase())) return true;
+ try {
+ const a = fs.statSync(left);
+ const b = fs.statSync(right);
+ return a.dev === b.dev && a.ino === b.ino;
+ } catch (_) {
+ return false;
+ }
+}
+
+function atomicWrite(file, bytes) {
+ const absolute = path.resolve(file);
+ const directory = path.dirname(absolute);
+ fs.mkdirSync(directory, { recursive: true });
+ const temporary = path.join(directory, `.${path.basename(absolute)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`);
+ let fd;
+ try {
+ fd = fs.openSync(temporary, 'wx', 0o600);
+ fs.writeFileSync(fd, bytes);
+ fs.fsyncSync(fd);
+ fs.closeSync(fd);
+ fd = undefined;
+ fs.renameSync(temporary, absolute);
+ try {
+ const dirFd = fs.openSync(directory, 'r');
+ try { fs.fsyncSync(dirFd); } finally { fs.closeSync(dirFd); }
+ } catch (_) { /* directory fsync is not available on every platform */ }
+ } finally {
+ if (fd !== undefined) fs.closeSync(fd);
+ try { fs.unlinkSync(temporary); } catch (_) { /* renamed or already removed */ }
+ }
+}
+
+let options;
+try {
+ options = parseArgs(process.argv.slice(2));
+} catch (error) {
+ console.error(error.message);
+ console.error(usage());
+ process.exit(2);
+}
+
+const inputPath = path.resolve(options.file);
+const outputPath = options.out ? path.resolve(options.out) : null;
+if (outputPath && sameFile(inputPath, outputPath)) {
+ console.error('--out must not be the same file as the input manifest (including symlink or hardlink aliases)');
+ process.exit(2);
+}
+
+try {
+ const report = readDataSurfaceManifest(inputPath, { requireReady: false });
+ const serialized = JSON.stringify(report, null, 2) + '\n';
+ if (outputPath) atomicWrite(outputPath, serialized);
+ else process.stdout.write(serialized);
+ if (report.reviewRequired && !options.allowReviewRequired) {
+ console.error(`Data Surface import blocked: ${report.blockers.join('; ')}`);
+ process.exit(1);
+ }
+} catch (error) {
+ console.error(error.message);
+ process.exit(2);
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-diff.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-diff.js
new file mode 100644
index 0000000..7308f01
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-diff.js
@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-diff.js — structural diff between two Data Packs (data.json).
+ *
+ * Usage:
+ * sg-data-pack diff Human-readable report
+ * sg-data-pack diff --json Machine-readable report
+ *
+ * Reports added/removed/changed entities, relations, aliases, contents, stages,
+ * sameAs links, and all v1.3 sections. Exit code: 0 = identical, 1 = differences,
+ * 2 = usage or input error.
+ */
+'use strict';
+const fs = require('node:fs');
+const { buildDiff, count } = require('./lib/sg-pack-diff.js');
+
+const [, , oldPath, newPath, ...flags] = process.argv;
+if (!oldPath || !newPath) {
+ console.error('Usage: sg-data-pack diff [--json]');
+ process.exit(2);
+}
+const asJson = flags.includes('--json');
+
+function load(file) {
+ try {
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
+ } catch (error) {
+ console.error(`Cannot read ${file}: ${error.message}`);
+ process.exit(2);
+ }
+}
+
+const report = buildDiff(load(oldPath), load(newPath), { old: oldPath, new: newPath });
+
+if (asJson) {
+ const { total, ...jsonReport } = report;
+ console.log(JSON.stringify(jsonReport, null, 2));
+} else {
+ const formatList = (label, section, showFields) => {
+ if (!count(section)) return;
+ console.log(`\n[${label}] +${section.added.length} -${section.removed.length} ~${section.changed.length}`);
+ if (section.added.length) console.log(' added: ' + section.added.join(', '));
+ if (section.removed.length) console.log(' removed: ' + section.removed.join(', '));
+ for (const change of section.changed.slice(0, showFields ? 50 : 10)) {
+ console.log(` changed: ${change.id}`);
+ if (showFields) for (const field of change.fields.slice(0, 8)) console.log(` .${field}`);
+ }
+ };
+ console.log(`sg-data-pack diff\n old: ${oldPath}\n new: ${newPath}`);
+ formatList('entities', report.entities, true);
+ formatList('relations', report.relations, true);
+ formatList('aliases', report.aliases, false);
+ formatList('contents', report.contents, true);
+ formatList('stages', report.stages, true);
+ if (report.stages.orderChanged) console.log(' order: changed');
+ if (report.sameAs.added.length || report.sameAs.removed.length) {
+ console.log(`\n[sameAs] +${report.sameAs.added.length} -${report.sameAs.removed.length}`);
+ report.sameAs.added.forEach((value) => console.log(' added: ' + value));
+ report.sameAs.removed.forEach((value) => console.log(' removed: ' + value));
+ }
+ formatList('derivations', report.derivations, true);
+ formatList('assets', report.assets, true);
+ formatList('provenance', report.provenance, false);
+ formatList('attributeTypes', report.attributeTypes, true);
+ formatList('relationTypes', report.relationTypes, true);
+ formatList('heroRelTypes', report.heroRelTypes, true);
+ formatList('kindNameFields', report.kindNameFields, true);
+ formatList('domain', report.domain, true);
+ formatList('meta', report.meta, true);
+ console.log(report.total ? `\n${report.total} difference(s) found.` : '\nNo structural differences.');
+ if (report.derivationsImpacted.length) {
+ console.log('\n⚠ derivations impact (blast radius):');
+ for (const derivation of report.derivationsImpacted) {
+ console.log(` • ${derivation.name} (${derivation.kind}) <- ${derivation.triggers.join(', ')}`);
+ if (derivation.note) console.log(` ${derivation.note}`);
+ }
+ }
+}
+process.exit(report.total ? 1 : 0);
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-evidence.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-evidence.js
new file mode 100644
index 0000000..f4d7907
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-evidence.js
@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+'use strict';
+const path = require('node:path');
+const { readRuntimeEvidence } = require('./lib/sg-runtime-evidence.js');
+const { readVisualEvidence } = require('./lib/sg-visual-evidence.js');
+
+const [, , kind, fileArg, ...args] = process.argv;
+if (!['runtime', 'visual'].includes(kind) || !fileArg || args.some((item) => item !== '--json') || args.filter((item) => item === '--json').length > 1) {
+ console.error('Usage: sg-data-pack evidence [--json]');
+ process.exit(2);
+}
+try {
+ const loaded = kind === 'runtime' ? readRuntimeEvidence(path.resolve(fileArg)) : readVisualEvidence(path.resolve(fileArg));
+ if (args.includes('--json')) process.stdout.write(JSON.stringify(loaded, null, 2) + '\n');
+ else {
+ console.log(`${kind} evidence: ${loaded.validation.status}`);
+ loaded.validation.errors.forEach((error) => console.error(' ' + error));
+ }
+ process.exit(loaded.validation.status === 'passed' ? 0 : loaded.validation.status === 'failed' || loaded.validation.status === 'not-assessed' ? 1 : 2);
+} catch (error) { console.error(`${kind} evidence: ${error.message}`); process.exit(2); }
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-experiment.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-experiment.js
new file mode 100644
index 0000000..69d8d89
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-experiment.js
@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+'use strict';
+const path = require('node:path');
+const { runExperiment } = require('./lib/sg-experiment.js');
+
+const [, , specArg, ...args] = process.argv;
+if (!specArg || specArg.startsWith('--')) { console.error('Usage: sg-data-pack experiment --out [--json]'); process.exit(2); }
+let out = null;
+let json = false;
+for (let index = 0; index < args.length; index += 1) {
+ const name = args[index];
+ if (name === '--json') { if (json) { console.error('Duplicate option: --json'); process.exit(2); } json = true; continue; }
+ if (name !== '--out') { console.error('Unknown option: ' + name); process.exit(2); }
+ if (out) { console.error('Duplicate option: --out'); process.exit(2); }
+ out = args[index + 1];
+ if (!out || out.startsWith('--')) { console.error('--out requires a value'); process.exit(2); }
+ index += 1;
+}
+if (!out) { console.error('experiment requires --out '); process.exit(2); }
+try {
+ const report = runExperiment({ specFile: path.resolve(specArg), outputRoot: path.resolve(out) });
+ if (json) process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+ else {
+ console.log(`Experiment ${report.experimentId}`);
+ console.log(`pass rate: ${report.summary.passRate === null ? 'N/A' : (report.summary.passRate * 100).toFixed(1) + '%'} (${report.summary.passed}/${report.summary.validTrials})`);
+ console.log(`claim level: ${report.summary.claimLevel}`);
+ console.log(`report: ${path.resolve(out, 'EXPERIMENT.md')}`);
+ }
+ process.exit(report.summary.infraErrors || report.summary.invalidTrials ? 3 : report.summary.passed === report.summary.validTrials ? 0 : 1);
+} catch (error) { console.error('experiment: ' + error.message); process.exit(2); }
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-extract.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-extract.js
new file mode 100644
index 0000000..677f65d
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-extract.js
@@ -0,0 +1,200 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-extract.js — SG Data Pack extraction / validation / equivalence testing
+ *
+ * Usage:
+ * node sg-pack-extract.js
+ * node sg-pack-extract.js --check
+ *
+ * The shared core defaults generated packs with: config.schemaVersion || '1.3'.
+ */
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { runExtraction, deepEqual } = require('./lib/sg-pack-extract-core.js');
+
+const [name, ...args] = process.argv.slice(2);
+const allowedFlags = new Set(['--check', '--compare-existing', '--check-output', '--force']);
+const seen = new Set();
+if (!name || name.startsWith('--')) {
+ console.error('Usage: node sg-pack-extract.js [--check] [--compare-existing|--check-output] [--force]');
+ process.exit(2);
+}
+for (const flag of args) {
+ if (!allowedFlags.has(flag)) {
+ console.error('Unknown option: ' + flag);
+ process.exit(2);
+ }
+ if (seen.has(flag)) {
+ console.error('Duplicate option: ' + flag);
+ process.exit(2);
+ }
+ seen.add(flag);
+}
+if (seen.has('--compare-existing') && seen.has('--check-output')) {
+ console.error('--compare-existing and --check-output are aliases; use only one');
+ process.exit(2);
+}
+const compareExisting = seen.has('--compare-existing') || seen.has('--check-output');
+const checkOnly = seen.has('--check') || compareExisting;
+const force = seen.has('--force');
+if (force && checkOnly) {
+ console.error('--force is only valid when writing extract outputs');
+ process.exit(2);
+}
+const configPath = path.resolve(process.cwd(), name.endsWith('.js') ? name : name + '.config.js');
+if (!fs.existsSync(configPath)) {
+ console.error('Config file not found: ' + configPath);
+ process.exit(2);
+}
+
+let extraction;
+try {
+ extraction = runExtraction(configPath);
+} catch (error) {
+ console.error('Extraction failed: ' + error.message);
+ process.exit(error.kind === 'input' ? 2 : 1);
+}
+const { config, defaults, pack, validation, equivalence } = extraction;
+
+console.log('== [' + config.libId + '] 1/4 Slicing engine default-data literals');
+for (const spec of config.literals) {
+ const value = defaults[spec.key];
+ const count = Array.isArray(value) ? value.length : Object.keys(value || {}).length;
+ console.log(' - ' + spec.key + ': ' + count + ' entries');
+}
+
+console.log('== 2/4 Building spec-compliant Data Pack');
+const missing = Object.entries(pack.assets || {}).filter(([, asset]) => !asset.exists);
+console.log(' - entities: ' + Object.keys(pack.entities || {}).length +
+ ', relations: ' + (pack.relations || []).length +
+ ', stages: ' + (pack.stages || []).length +
+ ', assets: ' + Object.keys(pack.assets || {}).length + (missing.length ? ' (missing ' + missing.length + ')' : ''));
+
+console.log('== 3/4 SGDataLoader validation' + (config.domainChecks ? ' + domain library-level checks' : ''));
+validation.errors.forEach((error) => console.error(' [ERROR] ' + error));
+validation.warnings.forEach((warning) => console.warn(' [warn] ' + warning));
+if (validation.errors.length) {
+ console.error('Validation failed; aborting.');
+ process.exit(1);
+}
+
+console.log('== 4/4 Equivalence test (fromPack(pack) vs engine defaults)');
+if (equivalence.diffs.length) {
+ equivalence.diffs.slice(0, 20).forEach((diff) => console.error(' [DIFF] ' + diff));
+ console.error('Equivalence test failed: ' + equivalence.diffs.length + ' differences; aborting.');
+ process.exit(1);
+}
+const coverage = equivalence.coverage || { extracted: [], mapped: [], ignored: [], unmapped: [], complete: false };
+console.log(' - all ' + equivalence.comparisons + ' deep comparisons passed; configured equivalence surface is lossless');
+console.log(' - coverage: ' + coverage.mapped.length + ' mapped, ' + coverage.ignored.length + ' explicitly ignored, ' + coverage.unmapped.length + ' unmapped');
+
+const dataDir = path.join(config.libDir, 'lib', 'data');
+const dataPath = path.join(dataDir, 'data.json');
+let existingDiffs = null;
+if (compareExisting) {
+ if (!fs.existsSync(dataPath)) {
+ console.log('== Existing output drift: NOT_ASSESSED (data.json does not exist)');
+ } else {
+ try {
+ const existing = JSON.parse(fs.readFileSync(dataPath, 'utf8'));
+ existingDiffs = deepEqual(existing, pack, '$', []);
+ } catch (error) {
+ console.error('Existing data.json cannot be compared: ' + error.message);
+ process.exit(2);
+ }
+ if (existingDiffs.length) {
+ existingDiffs.slice(0, 20).forEach((diff) => console.error(' [OUTPUT-DRIFT] ' + diff));
+ console.error('Existing output drift detected: ' + existingDiffs.length + ' differences');
+ process.exit(1);
+ }
+ console.log('== Existing output drift: passed (data.json matches the fresh in-memory pack)');
+ }
+}
+
+function assertOutputPathSafe(directory, files) {
+ if (fs.existsSync(directory) && fs.lstatSync(directory).isSymbolicLink()) throw new Error('data output directory must not be a symlink');
+ for (const file of files) {
+ if (fs.existsSync(file) && fs.lstatSync(file).isSymbolicLink()) throw new Error('extract output must not be a symlink: ' + file);
+ }
+}
+
+function atomicWriteSet(outputs) {
+ const temporary = outputs.map(({ file, bytes }) => ({
+ file,
+ bytes,
+ temp: path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`),
+ backup: null,
+ }));
+ const installed = [];
+ try {
+ for (const output of temporary) {
+ let fd;
+ try {
+ fd = fs.openSync(output.temp, 'wx', 0o600);
+ fs.writeFileSync(fd, output.bytes);
+ fs.fsyncSync(fd);
+ } finally {
+ if (fd !== undefined) fs.closeSync(fd);
+ }
+ }
+ for (const output of temporary) {
+ if (!fs.existsSync(output.file)) continue;
+ output.backup = `${output.file}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.bak`;
+ fs.renameSync(output.file, output.backup);
+ }
+ for (const output of temporary) {
+ fs.renameSync(output.temp, output.file);
+ installed.push(output.file);
+ }
+ for (const output of temporary) if (output.backup) fs.unlinkSync(output.backup);
+ } catch (error) {
+ for (const output of temporary) { try { if (fs.existsSync(output.temp)) fs.unlinkSync(output.temp); } catch (_) { /* absent */ } }
+ for (const file of installed) { try { if (fs.existsSync(file)) fs.unlinkSync(file); } catch (_) { /* absent */ } }
+ const rollbackErrors = [];
+ for (const output of temporary) {
+ if (!output.backup) continue;
+ try { if (fs.existsSync(output.backup)) fs.renameSync(output.backup, output.file); } catch (rollbackError) { rollbackErrors.push(rollbackError.message); }
+ }
+ if (rollbackErrors.length) throw new Error(error.message + '; rollback incomplete: ' + rollbackErrors.join('; '));
+ throw error;
+ }
+}
+
+if (!checkOnly) {
+ fs.mkdirSync(dataDir, { recursive: true });
+ const json = JSON.stringify(pack, null, 2) + '\n';
+ if (fs.existsSync(dataPath)) {
+ let existing;
+ try { existing = JSON.parse(fs.readFileSync(dataPath, 'utf8')); }
+ catch (error) { console.error('Existing data.json cannot be safely replaced: ' + error.message); process.exit(2); }
+ const diffs = deepEqual(existing, pack, '$', []);
+ if (diffs.length && !force) {
+ diffs.slice(0, 20).forEach((diff) => console.error(' [OUTPUT-DRIFT] ' + diff));
+ console.error('Refusing to overwrite an existing divergent data.json; review the drift or rerun with --force');
+ process.exit(1);
+ }
+ }
+
+ const schema = JSON.parse(fs.readFileSync(path.join(__dirname, 'lib', 'data-pack.schema.json'), 'utf8'));
+ schema.$id = 'https://sg.local/' + config.libId + '/data.schema.json';
+ schema.title = 'SG Data Pack v1.3 — ' + config.libId + ' data contract';
+ if (config.domainSchema) schema.properties.domain = config.domainSchema;
+ const outputs = [
+ { file: dataPath, bytes: json },
+ { file: path.join(dataDir, 'data.js'), bytes: '// @generated by sg-data-pack — do not edit by hand; regenerate from the reviewed Data Pack source\n' + 'globalThis.SG_DATA_PACK = ' + JSON.stringify(pack) + ';\n' },
+ { file: path.join(dataDir, 'data.schema.json'), bytes: JSON.stringify(schema, null, 2) + '\n' },
+ ];
+ try {
+ assertOutputPathSafe(dataDir, outputs.map((item) => item.file));
+ atomicWriteSet(outputs);
+ } catch (error) {
+ console.error('Could not install extract outputs: ' + error.message);
+ process.exit(2);
+ }
+
+ console.log('Written:');
+ outputs.forEach((output) => console.log(' - ' + output.file));
+}
+console.log('✔ [' + config.libId + '] done');
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-grade.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-grade.js
new file mode 100644
index 0000000..61d682a
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-grade.js
@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { gradeWorkspace } = require('./lib/sg-grader.js');
+
+const [, , specArg, workspaceArg, ...args] = process.argv;
+const usage = 'Usage: sg-data-pack grade [--artifacts dir] [--task-id id] [--tree sha256:...] [--out grade.json] [--json]';
+if (!specArg || !workspaceArg || specArg.startsWith('--') || workspaceArg.startsWith('--')) { console.error(usage); process.exit(2); }
+const values = new Set(['--artifacts', '--task-id', '--tree', '--out']);
+const booleans = new Set(['--json']);
+const options = {};
+for (let index = 0; index < args.length; index += 1) {
+ const name = args[index];
+ if (booleans.has(name)) { if (options[name]) { console.error(`Duplicate option: ${name}`); process.exit(2); } options[name] = true; continue; }
+ if (!values.has(name)) { console.error(`Unknown option: ${name}`); process.exit(2); }
+ const value = args[index + 1];
+ if (!value || value.startsWith('--')) { console.error(`${name} requires a value`); process.exit(2); }
+ if (options[name]) { console.error(`Duplicate option: ${name}`); process.exit(2); }
+ options[name] = value; index += 1;
+}
+try {
+ const spec = JSON.parse(fs.readFileSync(path.resolve(specArg), 'utf8'));
+ const report = gradeWorkspace({
+ spec, workspaceRoot: path.resolve(workspaceArg),
+ artifactRoot: options['--artifacts'] ? path.resolve(options['--artifacts']) : null,
+ taskId: options['--task-id'] || null, treeSha256: options['--tree'] || null,
+ });
+ const serialized = JSON.stringify(report, null, 2) + '\n';
+ if (options['--out']) fs.writeFileSync(path.resolve(options['--out']), serialized);
+ if (options['--json'] || !options['--out']) process.stdout.write(serialized);
+ else console.log(`grade: ${report.verdict} (${report.score}/${report.maximumScore})`);
+ process.exit(report.verdict === 'passed' ? 0 : 1);
+} catch (error) { console.error('grade: ' + error.message); process.exit(error.code === 'INVALID_GRADER_SPEC' ? 2 : 3); }
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-recrawl-skeleton.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-recrawl-skeleton.js
new file mode 100644
index 0000000..1dc26b0
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-recrawl-skeleton.js
@@ -0,0 +1,141 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-recrawl-skeleton.js - one-command recrawl normalization pipeline.
+ *
+ * Input: an existing Data Pack (baseline) + a list of crawled records (JSON
+ * array of {crawledName, ...fields} or newline-separated names).
+ * Output: a recrawl review skeleton:
+ * - hits: names that resolve via existing aliases (auto-confirmed)
+ * - misses: names with ranked alias candidates (awaiting confirmation)
+ * - crossCheck: for hits, compares crawled fields vs baseline (agree/conflict/gap)
+ * - review-report.json: agree / autoMergeable / needsHumanReview split
+ *
+ * Usage:
+ * sg-data-pack recrawl-skeleton [--out ] [--source ] [--fetchedAt ] [--candidate-ready --origin ]
+ *
+ * Exit: 0 = skeleton written, 1 = misses found (review needed), 2 = usage error.
+ *
+ * After review: extend pack.aliases with confirmed mappings, then run the gate.
+ */
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const review = require('./lib/sg-recrawl-review.js');
+
+const [, , packPath, recordsPath, ...rest] = process.argv;
+if (!packPath || !recordsPath) {
+ console.error('Usage: sg-data-pack recrawl-skeleton [--out ] [--source ] [--fetchedAt ]');
+ process.exit(2);
+}
+
+function flag(name) {
+ const index = rest.indexOf(name);
+ if (index < 0) return null;
+ if (index + 1 >= rest.length || rest[index + 1].startsWith('--')) {
+ console.error(`${name} requires a value`);
+ process.exit(2);
+ }
+ return rest[index + 1];
+}
+
+const outDir = flag('--out') || '.';
+const source = flag('--source') || '(unspecified)';
+const origin = flag('--origin');
+const fetchedAt = flag('--fetchedAt') || new Date().toISOString().slice(0, 10);
+const candidateReady = rest.includes('--candidate-ready');
+
+function loadJson(file, label) {
+ try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
+ catch (error) { console.error(label + ': ' + error.message); process.exit(2); }
+}
+
+let packBytes;
+let recordsBytes;
+try {
+ packBytes = fs.readFileSync(packPath);
+ recordsBytes = fs.readFileSync(recordsPath);
+} catch (error) {
+ console.error('Cannot read recrawl input: ' + error.message);
+ process.exit(2);
+}
+const pack = loadJson(packPath, 'Invalid data.json');
+const rawRecords = loadJson(recordsPath, 'Invalid records.json');
+let records;
+try {
+ records = review.normalizeRecords(rawRecords);
+} catch (error) {
+ console.error('Invalid records.json: ' + error.message);
+ process.exit(2);
+}
+
+const legacy = review.buildLegacyProjection(pack, rawRecords);
+const { summary, hits, misses, autoMergeable, needsHumanReview } = legacy;
+const report = {
+ source,
+ fetchedAt,
+ baseline: packPath,
+ summary,
+ hits,
+ misses,
+ autoMergeable,
+ needsHumanReview,
+};
+
+if (candidateReady) {
+ try {
+ report.candidateReview = review.buildCandidateReview({
+ pack,
+ rawRecords,
+ baselineBytes: packBytes,
+ recordsBytes,
+ origin,
+ sourceUrl: source,
+ fetchedAt,
+ });
+ } catch (error) {
+ console.error('Invalid candidate-ready options: ' + error.message);
+ process.exit(2);
+ }
+}
+
+function atomicWrite(file, content) {
+ fs.mkdirSync(path.dirname(file), { recursive: true });
+ const temp = path.join(
+ path.dirname(file),
+ `.${path.basename(file)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`,
+ );
+ let fd;
+ try {
+ fd = fs.openSync(temp, 'wx');
+ fs.writeFileSync(fd, content, 'utf8');
+ fs.fsyncSync(fd);
+ fs.closeSync(fd);
+ fd = undefined;
+ fs.renameSync(temp, file);
+ } finally {
+ if (fd !== undefined) fs.closeSync(fd);
+ try { fs.unlinkSync(temp); } catch (_) { /* already renamed or never created */ }
+ }
+}
+
+const reportPath = path.join(outDir, 'review-report.json');
+try {
+ atomicWrite(reportPath, JSON.stringify(report, null, 2) + '\n');
+} catch (error) {
+ console.error('Could not write review report: ' + error.message);
+ process.exit(2);
+}
+
+console.log(`recrawl-skeleton: ${records.length} records -> ${hits.length} hits, ${misses.length} misses`);
+console.log(` cross-check: ${report.autoMergeable.length} auto-mergeable (gap fill), ${report.needsHumanReview.length} needs human review (conflict)`);
+if (misses.length) {
+ console.log('\nmisses (extend pack.aliases after confirmation):');
+ for (const miss of misses) {
+ console.log(` ${miss.crawledName}`);
+ for (const candidate of miss.candidates) console.log(` ${candidate.score} ${candidate.id} (${candidate.matchedOn})`);
+ if (!miss.candidates.length) console.log(' (no candidates - likely a genuinely new entity)');
+ }
+}
+console.log(`\nwrote ${reportPath}`);
+process.exit(misses.length ? 1 : 0);
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-report.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-report.js
new file mode 100644
index 0000000..627b513
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-report.js
@@ -0,0 +1,729 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-report.js — unified user-facing Library Evolution Report.
+ *
+ * Usage:
+ * sg-data-pack report [--config config.js] [--baseline old.json]
+ * [--review review-report.json] [--audit candidate-audit.json]
+ * [--strict] [--verify-hash] [--asset-root dir] [--out report-dir] [--json]
+ */
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { spawnSync } = require('node:child_process');
+const { buildReport, diagnosticFinding, assurance, change, sha256 } = require('./lib/sg-run-report.js');
+const { inspectPack } = require('./lib/sg-pack-inspect.js');
+const { executeRules } = require('./lib/sg-pack-rules-core.js');
+const { buildDiff } = require('./lib/sg-pack-diff.js');
+const { ExtractionError } = require('./lib/sg-pack-extract-core.js');
+const { validateReportShape } = require('./lib/sg-pack-candidate.js');
+require('./lib/sg-data-loader.js');
+const loader = globalThis.SGDataLoader;
+const { sha256: reviewSha256, stableJson: reviewStableJson } = require('./lib/sg-recrawl-review.js');
+const { renderJson, renderTerminalSummary, renderMarkdown } = require('./lib/sg-report-renderer.js');
+
+const [, , libDirArg, ...args] = process.argv;
+const usage = 'Usage: sg-data-pack report [--config config.js] [--baseline old.json] [--review review-report.json] [--audit candidate-audit.json] [--strict] [--verify-hash] [--asset-root dir] [--out report-dir] [--json]';
+if (!libDirArg || libDirArg.startsWith('--')) {
+ console.error(usage);
+ process.exit(2);
+}
+
+const valueFlags = new Set(['--config', '--baseline', '--review', '--audit', '--asset-root', '--out']);
+const booleanFlags = new Set(['--strict', '--verify-hash', '--json']);
+const options = {};
+for (let index = 0; index < args.length; index += 1) {
+ const name = args[index];
+ if (booleanFlags.has(name)) {
+ if (Object.prototype.hasOwnProperty.call(options, name)) {
+ console.error(`Duplicate option: ${name}`);
+ process.exit(2);
+ }
+ options[name] = true;
+ continue;
+ }
+ if (!valueFlags.has(name)) {
+ console.error(`Unknown option: ${name}`);
+ process.exit(2);
+ }
+ if (Object.prototype.hasOwnProperty.call(options, name)) {
+ console.error(`Duplicate option: ${name}`);
+ process.exit(2);
+ }
+ const value = args[index + 1];
+ if (!value || value.startsWith('--')) {
+ console.error(`${name} requires a value`);
+ process.exit(2);
+ }
+ options[name] = value;
+ index += 1;
+}
+
+const libDir = path.resolve(libDirArg);
+const configPath = options['--config'] ? path.resolve(options['--config']) : null;
+const baselinePath = options['--baseline'] ? path.resolve(options['--baseline']) : null;
+const reviewPath = options['--review'] ? path.resolve(options['--review']) : null;
+const auditPath = options['--audit'] ? path.resolve(options['--audit']) : null;
+const outDir = options['--out'] ? path.resolve(options['--out']) : null;
+const assetRoot = options['--asset-root'] ? path.resolve(options['--asset-root']) : null;
+const strict = Boolean(options['--strict']);
+const verifyHash = Boolean(options['--verify-hash']);
+const asJson = Boolean(options['--json']);
+const dataFile = path.join(libDir, 'lib', 'data', 'data.json');
+const rulesFile = path.join(libDir, 'lib', 'data', 'data-rules.json');
+const guideFile = path.join(libDir, 'lib', 'data', 'DATA-GUIDE.md');
+
+function canonicalPath(file) {
+ let absolute = path.resolve(file);
+ const missing = [];
+ while (!fs.existsSync(absolute)) {
+ const parent = path.dirname(absolute);
+ if (parent === absolute) break;
+ missing.unshift(path.basename(absolute));
+ absolute = parent;
+ }
+ try { absolute = fs.realpathSync(absolute); } catch (_) { /* input reads report the concrete error */ }
+ return path.join(absolute, ...missing);
+}
+
+function sameFile(left, right) {
+ const a = canonicalPath(left);
+ const b = canonicalPath(right);
+ const insensitive = process.platform === 'darwin' || process.platform === 'win32';
+ if (a === b || (insensitive && a.toLowerCase() === b.toLowerCase())) return true;
+ try {
+ const leftStat = fs.statSync(left);
+ const rightStat = fs.statSync(right);
+ return leftStat.dev === rightStat.dev && leftStat.ino === rightStat.ino;
+ } catch (_) {
+ return false;
+ }
+}
+
+if (outDir) {
+ const outputs = [path.join(outDir, 'report.json'), path.join(outDir, 'REPORT.md')];
+ const inputs = [dataFile, rulesFile, guideFile, configPath, baselinePath, reviewPath, auditPath].filter(Boolean);
+ const libraryRoot = canonicalPath(libDir);
+ const outputRoot = canonicalPath(outDir);
+ const relativeLibraryOutput = path.relative(libraryRoot, outputRoot);
+ const insideLibrary = relativeLibraryOutput === '' || (!relativeLibraryOutput.startsWith('..' + path.sep) && relativeLibraryOutput !== '..' && !path.isAbsolute(relativeLibraryOutput));
+ if (insideLibrary) {
+ const suggestion = path.join(path.dirname(libraryRoot), path.basename(libraryRoot) + '-report');
+ console.error(`Report output is inside the source library:\n source: ${libraryRoot}\n output: ${outputRoot}\nUse a sibling directory, for example:\n --out ${suggestion}`);
+ process.exit(2);
+ }
+ if (sameFile(outputs[0], outputs[1])) {
+ console.error(`Report outputs resolve to the same file:\n ${canonicalPath(outputs[0])}\n ${canonicalPath(outputs[1])}`);
+ process.exit(2);
+ }
+ const collision = outputs.find((output) => inputs.some((input) => sameFile(output, input)));
+ if (collision) {
+ console.error(`Report output must not overwrite a consumed input:\n output: ${canonicalPath(collision)}`);
+ process.exit(2);
+ }
+}
+
+function safeRelative(file) {
+ return path.relative(process.cwd(), path.resolve(file)) || path.basename(file);
+}
+
+function readInput(file, label) {
+ let bytes;
+ try {
+ bytes = fs.readFileSync(file);
+ } catch (error) {
+ throw new Error(`${label} cannot be read: ${error.message}`);
+ }
+ try {
+ return { value: JSON.parse(bytes.toString('utf8')), bytes, digest: sha256(bytes) };
+ } catch (error) {
+ throw new Error(`${label} is not valid JSON: ${error.message}`);
+ }
+}
+
+function digestFile(file, label) {
+ try {
+ return sha256(fs.readFileSync(file));
+ } catch (error) {
+ throw new Error(`${label} cannot be read: ${error.message}`);
+ }
+}
+
+function recordConsumed(inputs, file, digest) {
+ if (!inputs.consumedFiles) inputs.consumedFiles = {};
+ inputs.consumedFiles[canonicalPath(file)] = digest;
+}
+
+function runExtractionIsolated(configPath, expectedLibDir, expectedLibId) {
+ const result = spawnSync(process.execPath, [
+ path.join(__dirname, 'lib', 'sg-pack-extract-report-worker.js'),
+ configPath,
+ expectedLibDir,
+ expectedLibId || '',
+ ], {
+ encoding: 'utf8',
+ stdio: ['ignore', 'pipe', 'pipe', 'pipe'],
+ maxBuffer: 32 * 1024 * 1024,
+ });
+ if (result.error) throw new ExtractionError(result.error.message, 'input');
+ if (result.stdout) process.stderr.write(result.stdout);
+ if (result.stderr) process.stderr.write(result.stderr);
+ const payloadText = result.output && result.output[3] ? String(result.output[3]) : '';
+ if (!payloadText) throw new ExtractionError('extraction worker returned no structured result', 'gate');
+ let payload;
+ try { payload = JSON.parse(payloadText); }
+ catch (error) { throw new ExtractionError(`extraction worker returned invalid result: ${error.message}`, 'gate'); }
+ if (!payload.ok) throw new ExtractionError(payload.error.message, payload.error.kind || 'gate');
+ return payload.result;
+}
+
+function addValidationFindings(findings, validation, phase, strictMode) {
+ for (const error of validation.errors || []) {
+ findings.push(diagnosticFinding({ value: error, phase, severity: 'error', category: 'contract', blocking: true }));
+ }
+ for (const warning of validation.warnings || []) {
+ findings.push(diagnosticFinding({ value: warning, phase, severity: 'warning', category: 'contract', blocking: strictMode }));
+ }
+}
+
+function addAssetFindings(findings, assets) {
+ for (const item of assets.mismatches || []) {
+ findings.push(diagnosticFinding({
+ value: `asset hash mismatch: ${item.path}`,
+ code: 'ASSET_HASH_MISMATCH', phase: 'validate', category: 'asset', severity: 'error', blocking: true,
+ title: '资源文件与清单 hash 不一致', subject: item.path, expected: item.expected, actual: item.actual, evidence: item,
+ repairHint: '确认文件来源后重新生成资产清单;不要在未复核内容的情况下只改 hash。',
+ }));
+ }
+ for (const item of assets.missing || []) {
+ findings.push(diagnosticFinding({
+ value: `asset verification failed: ${item.path} (${item.error})`,
+ code: 'ASSET_UNREADABLE', phase: 'validate', category: 'asset', severity: 'error', blocking: true,
+ title: '资源文件无法校验', subject: item.path, expected: item.expected, evidence: item,
+ repairHint: '修正 assets 路径或补齐文件,再重新运行 --verify-hash。',
+ }));
+ }
+}
+
+function findingCode(classification) {
+ return {
+ conflict: 'RECRAWL_CONFLICT', gap: 'RECRAWL_GAP', miss: 'RECRAWL_MISS', unsupported: 'RECRAWL_UNSUPPORTED',
+ }[classification] || 'RECRAWL_REVIEW';
+}
+
+function addReviewFinding(findings, risks, item, resolved) {
+ const classification = item.classification || 'review';
+ const itemId = item.itemId || `legacy:${classification}:${item.index}`;
+ const status = resolved.has(itemId) ? 'resolved' : 'review-required';
+ findings.push(diagnosticFinding({
+ value: `${classification} requires an explicit review decision`,
+ code: findingCode(classification), phase: 'review', category: classification === 'miss' ? 'identity' : 'review',
+ severity: 'warning', status, blocking: status === 'review-required',
+ title: status === 'resolved' ? `${classification} 已完成人工复核` : `${classification} 待人工复核`,
+ subject: itemId, evidence: item,
+ }));
+ if (status !== 'resolved') {
+ risks.push({
+ id: `risk-${itemId}`, severity: 'high', status: 'open', title: `${classification} review item`,
+ reason: `review item ${itemId} 尚未被有效 Candidate audit 覆盖。`, itemId,
+ });
+ }
+ return status;
+}
+
+function validateCandidateReview(review) {
+ if (!review || typeof review !== 'object' || review.version !== '1.0') throw new Error('candidateReview.version must be "1.0"');
+ if (!Array.isArray(review.observations) || !Array.isArray(review.reviewItems)) throw new Error('candidateReview observations and reviewItems must be arrays');
+ if (typeof review.reportId !== 'string') throw new Error('candidateReview.reportId is required');
+ for (const field of ['baselineSha256', 'recordsSha256']) {
+ if (typeof review[field] !== 'string' || !/^sha256:[0-9a-f]{64}$/.test(review[field])) throw new Error(`candidateReview.${field} must be a SHA-256 digest`);
+ }
+ const { reportId, ...withoutId } = review;
+ if (reportId !== reviewSha256(reviewStableJson(withoutId))) throw new Error('candidateReview.reportId does not match report contents');
+ const ids = review.reviewItems.map((item) => item && item.itemId);
+ if (ids.some((id) => typeof id !== 'string' || !id) || new Set(ids).size !== ids.length) throw new Error('candidateReview contains invalid or duplicate itemId values');
+ try { validateReportShape({ candidateReview: review }); }
+ catch (error) { throw new Error(`candidateReview shape is invalid: ${error.message}`); }
+ return review;
+}
+
+function collectReview(report, resolved, findings, risks) {
+ let items;
+ if (report.candidateReview) {
+ const review = validateCandidateReview(report.candidateReview);
+ items = review.reviewItems;
+ } else {
+ if (!Array.isArray(report.misses) || !Array.isArray(report.autoMergeable) || !Array.isArray(report.needsHumanReview)) {
+ throw new Error('review report must contain candidateReview or the legacy misses/autoMergeable/needsHumanReview arrays');
+ }
+ items = [];
+ report.misses.forEach((entry, index) => items.push({ itemId: `legacy:miss:${index}`, index, kind: 'identity', classification: 'miss', ...entry }));
+ report.autoMergeable.forEach((entry, recordIndex) => (entry.gaps || []).forEach((gap, fieldIndex) => items.push({ itemId: `legacy:gap:${recordIndex}:${fieldIndex}`, index: `${recordIndex}:${fieldIndex}`, kind: 'field', classification: 'gap', entityId: entry.id, ...gap })));
+ report.needsHumanReview.forEach((entry, recordIndex) => (entry.conflicts || []).forEach((conflict, fieldIndex) => items.push({ itemId: `legacy:conflict:${recordIndex}:${fieldIndex}`, index: `${recordIndex}:${fieldIndex}`, kind: 'field', classification: 'conflict', entityId: entry.id, ...conflict })));
+ }
+ const statuses = items.map((item) => addReviewFinding(findings, risks, item, resolved));
+ const open = statuses.filter((status) => status === 'review-required').length;
+ const classifications = items.reduce((counts, item) => {
+ counts[item.classification] = (counts[item.classification] || 0) + 1;
+ return counts;
+ }, {});
+ return {
+ assurance: assurance('crawl-review', 'Crawl review', open ? 'review-required' : 'passed', {
+ blocking: Boolean(open), evidence: `${items.length} review items / ${open} unresolved`, details: classifications,
+ }),
+ total: items.length,
+ open,
+ reportId: report.candidateReview && report.candidateReview.reportId,
+ baselineSha256: report.candidateReview && report.candidateReview.baselineSha256,
+ };
+}
+
+function validateAudit(audit, strictMode, expected = {}) {
+ if (!audit || typeof audit !== 'object' || audit.auditVersion !== '1.0') throw new Error('candidate audit must use auditVersion "1.0"');
+ if (!audit.inputs || typeof audit.inputs !== 'object') throw new Error('candidate audit inputs are required');
+ for (const field of ['baselineSha256', 'recordsSha256', 'reportId', 'decisionsSha256', 'candidateSha256']) {
+ if (typeof audit.inputs[field] !== 'string' || !/^sha256:[0-9a-f]{64}$/.test(audit.inputs[field])) throw new Error(`candidate audit inputs.${field} must be a SHA-256 digest`);
+ }
+ if (expected.candidateSha256 && audit.inputs.candidateSha256 !== expected.candidateSha256) throw new Error('candidate audit candidateSha256 does not match current Data Pack');
+ if (expected.baselineSha256 && audit.inputs.baselineSha256 !== expected.baselineSha256) throw new Error('candidate audit baselineSha256 does not match reviewed baseline');
+ if (expected.reportId && audit.inputs.reportId !== expected.reportId) throw new Error('candidate audit reportId does not match supplied review report');
+ if (!audit.validation || !Array.isArray(audit.validation.errors) || !Array.isArray(audit.validation.warnings)) throw new Error('candidate audit validation errors/warnings must be arrays');
+ if (!Array.isArray(audit.operations) || !Array.isArray(audit.unresolved) || !Array.isArray(audit.derivationsImpacted)) throw new Error('candidate audit operations/unresolved/derivationsImpacted must be arrays');
+ const allowedResults = new Set(['applied', 'noop-already-applied', 'kept-baseline', 'rejected']);
+ const operationIds = new Set();
+ for (const operation of audit.operations) {
+ if (!operation || typeof operation.itemId !== 'string' || !operation.itemId || !allowedResults.has(operation.result)) throw new Error('candidate audit contains an invalid operation');
+ if (operationIds.has(operation.itemId)) throw new Error(`candidate audit contains a duplicate operation: ${operation.itemId}`);
+ operationIds.add(operation.itemId);
+ }
+ const unresolvedIds = new Set();
+ for (const unresolved of audit.unresolved) {
+ const itemId = typeof unresolved === 'string' ? unresolved : unresolved && unresolved.itemId;
+ if (typeof itemId !== 'string' || !itemId) throw new Error('candidate audit contains an invalid unresolved item');
+ if (unresolvedIds.has(itemId) || operationIds.has(itemId)) throw new Error(`candidate audit item appears more than once: ${itemId}`);
+ unresolvedIds.add(itemId);
+ }
+ if (expected.itemIds) {
+ const expectedIds = new Set(expected.itemIds);
+ const observedIds = new Set([...operationIds, ...unresolvedIds]);
+ if (expectedIds.size !== observedIds.size || [...expectedIds].some((itemId) => !observedIds.has(itemId))) {
+ throw new Error('candidate audit operations/unresolved do not exactly cover review items');
+ }
+ }
+ const failed = audit.status !== 'valid' || audit.validation.errors.length > 0 || unresolvedIds.size > 0 || (strictMode && audit.validation.warnings.length > 0);
+ return { failed, resolved: new Set(failed ? [] : operationIds) };
+}
+
+function stageMap(stages) {
+ return Object.fromEntries((stages || []).map((stage, index) => [stage && (stage.key || stage.id) || `#${index}`, stage]));
+}
+
+function pairMap(pairs) {
+ return Object.fromEntries((pairs || []).map((pair) => [JSON.stringify([...pair].sort()), pair]));
+}
+
+function relationKey(relation) {
+ if (relation && relation.id) return `id:${relation.id}`;
+ const scope = Array.isArray(relation && relation.scope) ? [...new Set(relation.scope)].sort().join('|') : '*';
+ return `${relation && relation.a}::${relation && relation.b}::${relation && relation.type}::scope=${scope}`;
+}
+
+function relationMap(relations) {
+ const map = {};
+ const seen = {};
+ for (const relation of relations || []) {
+ const base = relationKey(relation);
+ seen[base] = (seen[base] || 0) + 1;
+ map[seen[base] === 1 ? base : `${base}#${seen[base]}`] = relation;
+ }
+ return map;
+}
+
+function sectionMap(pack, section) {
+ if (section === 'relations') return relationMap(pack.relations);
+ if (section === 'stages') return stageMap(pack.stages);
+ if (section === 'sameAs') return pairMap(pack.sameAs);
+ return pack[section] && typeof pack[section] === 'object' && !Array.isArray(pack[section]) ? pack[section] : {};
+}
+
+function collectChanges(diff, oldPack, newPack) {
+ const changes = [];
+ for (const [section, sectionDiff] of Object.entries(diff)) {
+ if (!sectionDiff || typeof sectionDiff !== 'object' || !Array.isArray(sectionDiff.added)) continue;
+ const beforeMap = sectionMap(oldPack, section);
+ const afterMap = sectionMap(newPack, section);
+ for (const id of sectionDiff.added) changes.push(change({ id: `add-${section}-${id}`, section, kind: 'added', title: `新增 ${section} 记录`, subject: id, after: afterMap[id] === undefined ? id : afterMap[id] }));
+ for (const id of sectionDiff.removed) changes.push(change({ id: `remove-${section}-${id}`, section, kind: 'removed', title: `移除 ${section} 记录`, subject: id, before: beforeMap[id] === undefined ? id : beforeMap[id] }));
+ for (const item of sectionDiff.changed || []) changes.push(change({ id: `change-${section}-${item.id}`, section, kind: 'updated', title: `更新 ${section} 记录`, subject: item.id, before: beforeMap[item.id], after: afterMap[item.id], fields: item.fields }));
+ }
+ if (diff.stages && diff.stages.orderChanged) {
+ changes.push(change({
+ id: 'reorder-stages', section: 'stages', kind: 'reordered', title: '调整 stage 顺序', subject: 'stages',
+ before: (oldPack.stages || []).map((stage, index) => stage && (stage.key || stage.id) || `#${index}`),
+ after: (newPack.stages || []).map((stage, index) => stage && (stage.key || stage.id) || `#${index}`),
+ }));
+ }
+ return changes;
+}
+
+function reportMode() {
+ if (auditPath) return 'candidate';
+ if (reviewPath) return 'review';
+ if (baselinePath) return 'evolution';
+ if (configPath) return 'integration';
+ return 'pack';
+}
+
+function collectReport() {
+ const findings = [];
+ const changes = [];
+ const assurances = [];
+ const risks = [];
+ const operations = [];
+ const impacts = [];
+ const artifacts = [];
+ const raw = {};
+ const inputs = { strict, verifyHash };
+ let pack = {};
+ let packInspection = null;
+ let libId = path.basename(libDir);
+ let inputError = false;
+ let baselineInput = null;
+ let reviewInput = null;
+ let auditInput = null;
+ let auditState = { failed: false, resolved: new Set() };
+
+ try {
+ if (!fs.existsSync(libDir) || !fs.statSync(libDir).isDirectory()) throw new Error(`library directory not found: ${libDir}`);
+ packInspection = inspectPack({ dataFile, verifyHash, assetRoot });
+ pack = packInspection.pack;
+ libId = pack && pack.meta && pack.meta.id || libId;
+ inputs.dataFile = safeRelative(dataFile);
+ inputs.dataSha256 = packInspection.digest;
+ if (assetRoot) inputs.assetRoot = safeRelative(assetRoot);
+ inputs.consumedFiles = Object.fromEntries(Object.entries(packInspection.consumedFiles || {}).map(([file, digest]) => [canonicalPath(file), digest]));
+ raw.validation = packInspection.validation;
+ raw.assets = packInspection.assets;
+ addValidationFindings(findings, packInspection.validation, 'validate', strict);
+ addAssetFindings(findings, packInspection.assets);
+ const contractFailed = packInspection.validation.errors.length > 0 || (strict && packInspection.validation.warnings.length > 0);
+ assurances.push(assurance('data-pack-contract', 'Data Pack contract', contractFailed ? 'failed' : 'passed', {
+ evidence: `SGDataLoader; ${packInspection.validation.errors.length} errors / ${packInspection.validation.warnings.length} warnings${strict ? ' (strict)' : ''}`,
+ details: { errors: packInspection.validation.errors.length, warnings: packInspection.validation.warnings.length, strict }, blocking: contractFailed,
+ }));
+ const missingManifestHashes = packInspection.assets.skipped.filter((item) => item.reason === 'missing-manifest-hash');
+ const assetFailed = verifyHash && (packInspection.assets.mismatches.length > 0 || packInspection.assets.missing.length > 0);
+ const assetUnassessed = verifyHash && missingManifestHashes.length > 0;
+ if (assetUnassessed) {
+ risks.push({ id: 'risk-asset-manifest-hash', severity: 'medium', status: 'open', title: '部分资源没有 manifest hash', reason: `${missingManifestHashes.length} 个资源没有可比对的 SHA-1,完整资源完整性未评估。`, assets: missingManifestHashes });
+ }
+ assurances.push(assurance('asset-integrity', 'Asset integrity', !verifyHash ? 'not-assessed' : assetFailed ? 'failed' : assetUnassessed ? 'not-assessed' : 'passed', {
+ evidence: verifyHash ? `${packInspection.assets.summary.checked} files checked` : 'verify hash not requested',
+ details: packInspection.assets.summary, blocking: assetFailed,
+ }));
+ artifacts.push({ id: 'data-json', type: 'input', path: safeRelative(dataFile), digest: packInspection.digest });
+ } catch (error) {
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'validate', category: 'contract', severity: 'error', blocking: true, code: 'INPUT_ERROR', title: '无法读取 Data Pack' }));
+ assurances.push(assurance('data-pack-contract', 'Data Pack contract', 'failed', { blocking: true, evidence: error.message }));
+ assurances.push(assurance('asset-integrity', 'Asset integrity', 'not-assessed', { blocking: false, evidence: 'Data Pack input unavailable' }));
+ }
+
+ if (baselinePath) {
+ try {
+ baselineInput = readInput(baselinePath, 'baseline Data Pack');
+ const baselineValidation = loader.validate(baselineInput.value);
+ if (baselineValidation.errors.length || (strict && baselineValidation.warnings.length)) {
+ throw new Error(`baseline Data Pack is invalid: ${[...baselineValidation.errors, ...(strict ? baselineValidation.warnings : [])].join('; ')}`);
+ }
+ if (packInspection && baselineInput.value.meta.id !== pack.meta.id) {
+ throw new Error(`baseline meta.id does not match current Data Pack: ${baselineInput.value.meta.id}`);
+ }
+ inputs.baseline = safeRelative(baselinePath);
+ inputs.baselineSha256 = baselineInput.digest;
+ recordConsumed(inputs, baselinePath, baselineInput.digest);
+ artifacts.push({ id: 'baseline-data', type: 'input', path: safeRelative(baselinePath), digest: baselineInput.digest });
+ } catch (error) {
+ baselineInput = null;
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'diff', category: 'contract', severity: 'error', blocking: true, code: 'DIFF_INPUT_ERROR', title: '无法读取 baseline Data Pack' }));
+ }
+ }
+ if (reviewPath) {
+ try {
+ reviewInput = readInput(reviewPath, 'review report');
+ inputs.review = safeRelative(reviewPath);
+ inputs.reviewSha256 = reviewInput.digest;
+ recordConsumed(inputs, reviewPath, reviewInput.digest);
+ artifacts.push({ id: 'review-report', type: 'input', path: safeRelative(reviewPath), digest: reviewInput.digest });
+ } catch (error) {
+ reviewInput = null;
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'review', category: 'review', severity: 'error', blocking: true, code: 'REVIEW_INPUT_ERROR', title: '无法读取 review report' }));
+ }
+ }
+ if (auditPath) {
+ try {
+ auditInput = readInput(auditPath, 'candidate audit');
+ auditState = validateAudit(auditInput.value, strict, {
+ candidateSha256: packInspection && packInspection.digest,
+ baselineSha256: baselineInput && baselineInput.digest,
+ reportId: reviewInput && reviewInput.value && reviewInput.value.candidateReview && reviewInput.value.candidateReview.reportId,
+ itemIds: reviewInput && reviewInput.value && reviewInput.value.candidateReview && reviewInput.value.candidateReview.reviewItems
+ ? reviewInput.value.candidateReview.reviewItems.map((item) => item.itemId)
+ : null,
+ });
+ inputs.audit = safeRelative(auditPath);
+ inputs.auditSha256 = auditInput.digest;
+ recordConsumed(inputs, auditPath, auditInput.digest);
+ artifacts.push({ id: 'candidate-audit', type: 'input', path: safeRelative(auditPath), digest: auditInput.digest });
+ } catch (error) {
+ auditInput = null;
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'candidate', category: 'review', severity: 'error', blocking: true, code: 'AUDIT_INPUT_ERROR', title: '无法读取 Candidate audit' }));
+ }
+ }
+
+ if (packInspection && fs.existsSync(rulesFile)) {
+ try {
+ const rulesInput = readInput(rulesFile, 'data-rules.json');
+ inputs.rules = safeRelative(rulesFile);
+ inputs.rulesSha256 = rulesInput.digest;
+ recordConsumed(inputs, rulesFile, rulesInput.digest);
+ artifacts.push({ id: 'data-rules', type: 'input', path: safeRelative(rulesFile), digest: rulesInput.digest });
+ const rulesResult = executeRules(rulesInput.value, pack);
+ raw.rules = rulesResult;
+ for (const error of rulesResult.structuralErrors) findings.push(diagnosticFinding({ value: error, phase: 'rules', category: 'rule', severity: 'error', blocking: true, code: 'RULES_STRUCTURE', title: '规则文件结构错误' }));
+ for (const result of rulesResult.results) {
+ if (result.status === 'failed' || result.status === 'error') {
+ const isBlocking = result.level === 'hard' || result.status === 'error' || strict;
+ findings.push(diagnosticFinding({
+ value: result.exception ? result.exception.message : result.rule, phase: 'rules', category: 'rule',
+ severity: result.level === 'hard' || result.status === 'error' ? 'error' : 'warning', blocking: isBlocking,
+ code: result.id, title: result.status === 'error' ? '规则执行错误' : '库级规则未通过', subject: result.subject,
+ evidence: result.evidence, repairHint: result.repairHint,
+ }));
+ }
+ if (result.status === 'no-check') risks.push({ id: `risk-rule-${result.id}`, severity: 'medium', status: 'open', title: `规则 ${result.id} 没有 executable check`, reason: '规则只能作为文档提示,不能被自动验证。' });
+ }
+ for (const [index, note] of ((rulesResult.profile && rulesResult.profile.dataShapeNotes) || []).entries()) {
+ risks.push({ id: `risk-library-boundary-${index + 1}`, severity: 'low', status: 'monitored', title: '组件库开发边界', reason: note, source: 'data-rules.json profile.dataShapeNotes' });
+ }
+ const rulesFailed = rulesResult.structuralErrors.length > 0 || rulesResult.counts.hardFail > 0 || rulesResult.counts.errors > 0 || (strict && rulesResult.counts.softFail > 0);
+ assurances.push(assurance('library-rules', 'Library rules', rulesFailed ? 'failed' : 'passed', {
+ evidence: `${rulesResult.counts.passed}/${rulesResult.counts.total} executable rules passed`, details: rulesResult.counts, blocking: rulesFailed,
+ }));
+ } catch (error) {
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'rules', category: 'rule', severity: 'error', blocking: true, code: 'RULES_INPUT_ERROR', title: '无法读取 library rules' }));
+ assurances.push(assurance('library-rules', 'Library rules', 'failed', { blocking: true, evidence: error.message }));
+ }
+ } else {
+ assurances.push(assurance('library-rules', 'Library rules', 'not-assessed', { blocking: false, evidence: packInspection ? 'data-rules.json not found' : 'Data Pack input unavailable' }));
+ risks.push({ id: 'risk-no-rules', severity: 'medium', status: 'open', title: 'Library rules 未评估', reason: packInspection ? '组件库特定约束没有被自动评估。' : 'Data Pack 输入不可用。' });
+ }
+
+ if (fs.existsSync(guideFile)) {
+ try {
+ const digest = digestFile(guideFile, 'DATA-GUIDE.md');
+ inputs.dataGuide = safeRelative(guideFile);
+ inputs.dataGuideSha256 = digest;
+ recordConsumed(inputs, guideFile, digest);
+ artifacts.push({ id: 'data-guide', type: 'input', path: safeRelative(guideFile), digest });
+ } catch (error) {
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'coverage', category: 'contract', severity: 'error', blocking: true, code: 'GUIDE_INPUT_ERROR', title: '无法读取 DATA-GUIDE' }));
+ }
+ }
+
+ if (configPath) {
+ try {
+ const configDigest = digestFile(configPath, 'extract config');
+ inputs.config = safeRelative(configPath);
+ inputs.configSha256 = configDigest;
+ recordConsumed(inputs, configPath, configDigest);
+ artifacts.push({ id: 'extract-config', type: 'input', path: safeRelative(configPath), digest: configDigest });
+ const extraction = runExtractionIsolated(configPath, libDir, libId);
+ const extractionFiles = Object.fromEntries(Object.entries(extraction.consumedFiles || {}).sort(([left], [right]) => left.localeCompare(right)));
+ inputs.consumedFiles = Object.fromEntries(Object.entries({ ...inputs.consumedFiles, ...extractionFiles }).sort(([left], [right]) => left.localeCompare(right)));
+ inputs.extractionFiles = extractionFiles;
+ raw.extraction = { inventory: extraction.inventory, validation: extraction.validation, equivalence: extraction.equivalence, consumedFiles: extraction.consumedFiles };
+ addValidationFindings(findings, extraction.validation, 'extract', strict);
+ if (extraction.equivalence.diffs.length) findings.push(diagnosticFinding({
+ value: `__fromPack equivalence has ${extraction.equivalence.diffs.length} differences`, phase: 'extract', category: 'equivalence', severity: 'error', blocking: true,
+ code: 'EQUIVALENCE', title: '引擎数据无法无损还原', evidence: extraction.equivalence.diffs,
+ }));
+ const extractionFailed = extraction.validation.errors.length > 0 || extraction.equivalence.diffs.length > 0 || (strict && extraction.validation.warnings.length > 0);
+ assurances.push(assurance('extract-equivalence', 'Extract / deep equivalence', extractionFailed ? 'failed' : 'passed', {
+ evidence: `${extraction.equivalence.comparisons} deep comparisons`, details: { diffs: extraction.equivalence.diffs.length, warnings: extraction.validation.warnings.length }, blocking: extractionFailed,
+ }));
+ } catch (error) {
+ const isInputError = !(error instanceof ExtractionError) || error.kind === 'input';
+ if (isInputError) inputError = true;
+ findings.push(diagnosticFinding({
+ value: error, phase: 'extract', category: 'equivalence', severity: 'error', blocking: true,
+ code: isInputError ? 'EXTRACT_INPUT_ERROR' : 'EQUIVALENCE_RUNTIME_ERROR',
+ title: isInputError ? 'extract 配置或输入无效' : 'extract/equivalence 执行失败',
+ }));
+ assurances.push(assurance('extract-equivalence', 'Extract / deep equivalence', 'failed', { blocking: true, evidence: error.message }));
+ }
+ } else {
+ assurances.push(assurance('extract-equivalence', 'Extract / deep equivalence', 'not-assessed', { blocking: false, evidence: 'extract config not provided' }));
+ }
+
+ if (baselineInput && packInspection) {
+ try {
+ const diff = buildDiff(baselineInput.value, pack, { old: baselinePath, new: dataFile });
+ raw.diff = diff;
+ changes.push(...collectChanges(diff, baselineInput.value, pack));
+ impacts.push(...(diff.derivationsImpacted || []));
+ assurances.push(assurance('baseline-diff', 'Baseline diff', 'passed', { evidence: `${diff.total} structural differences`, details: { total: diff.total }, blocking: false }));
+ } catch (error) {
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'diff', category: 'contract', severity: 'error', blocking: true, code: 'DIFF_INPUT_ERROR', title: '无法生成 baseline diff' }));
+ assurances.push(assurance('baseline-diff', 'Baseline diff', 'failed', { blocking: true, evidence: error.message }));
+ }
+ } else {
+ assurances.push(assurance('baseline-diff', 'Baseline diff', baselinePath ? 'failed' : 'not-assessed', { blocking: Boolean(baselinePath), evidence: baselinePath ? 'baseline or current Data Pack input unavailable' : 'baseline not provided' }));
+ }
+
+ let auditValid = false;
+ if (auditInput) {
+ const audit = auditInput.value;
+ raw.audit = audit;
+ operations.push(...audit.operations);
+ impacts.push(...audit.derivationsImpacted);
+ for (const operation of audit.operations) {
+ changes.push(change({
+ id: `operation-${operation.itemId}`, section: 'candidate', kind: operation.result, title: `Candidate ${operation.result}`,
+ subject: operation.target, before: operation.before, after: operation.after, evidence: operation.provenance || operation.note,
+ }));
+ }
+ for (const error of audit.validation.errors) findings.push(diagnosticFinding({ value: error, phase: 'candidate', category: 'contract', severity: 'error', blocking: true, code: 'CANDIDATE_VALIDATION', title: 'Candidate validation 未通过' }));
+ for (const warning of audit.validation.warnings) findings.push(diagnosticFinding({ value: warning, phase: 'candidate', category: 'contract', severity: 'warning', blocking: strict, code: 'CANDIDATE_WARNING', title: 'Candidate validation warning' }));
+ auditValid = !auditState.failed;
+ assurances.push(assurance('candidate-audit', 'Candidate audit', auditValid ? 'passed' : 'failed', {
+ evidence: `${audit.operations.length} operations / ${audit.unresolved.length} unresolved`, details: audit.validation, blocking: !auditValid,
+ }));
+ } else {
+ assurances.push(assurance('candidate-audit', 'Candidate audit', auditPath ? 'failed' : 'not-assessed', { blocking: Boolean(auditPath), evidence: auditPath ? 'candidate audit input unavailable' : 'candidate audit not provided' }));
+ }
+
+ if (reviewInput) {
+ try {
+ const reviewResult = collectReview(reviewInput.value, auditState.resolved, findings, risks);
+ if (auditInput && reviewResult.reportId && auditInput.value.inputs.reportId !== reviewResult.reportId) {
+ throw new Error('candidate audit reportId does not match the supplied review report');
+ }
+ const expectedReviewBaseline = baselineInput
+ ? baselineInput.digest
+ : auditInput
+ ? auditInput.value.inputs.baselineSha256
+ : packInspection && packInspection.digest;
+ if (reviewResult.baselineSha256 && expectedReviewBaseline && reviewResult.baselineSha256 !== expectedReviewBaseline) {
+ throw new Error('candidateReview baselineSha256 does not match the reviewed Data Pack bytes');
+ }
+ raw.review = reviewInput.value;
+ raw.reviewSummary = { total: reviewResult.total, unresolved: reviewResult.open };
+ assurances.push(reviewResult.assurance);
+ } catch (error) {
+ inputError = true;
+ findings.push(diagnosticFinding({ value: error, phase: 'review', category: 'review', severity: 'error', blocking: true, code: 'REVIEW_INPUT_ERROR', title: 'review report 与输入不一致' }));
+ assurances.push(assurance('crawl-review', 'Crawl review', 'failed', { blocking: true, evidence: error.message }));
+ }
+ } else {
+ assurances.push(assurance('crawl-review', 'Crawl review', reviewPath ? 'failed' : 'not-assessed', { blocking: Boolean(reviewPath), evidence: reviewPath ? 'review report input unavailable' : 'review report not provided' }));
+ }
+
+ assurances.push(assurance('runtime-mount', 'Runtime DOM mount', 'not-assessed', { blocking: false, evidence: 'browser/runtime mount test not connected to report collector' }));
+ assurances.push(assurance('visual-regression', 'Visual regression', 'not-assessed', { blocking: false, evidence: 'visual regression test not connected to report collector' }));
+ assurances.push(assurance('production-crawl', 'Production crawl', 'not-assessed', { blocking: false, evidence: reviewInput ? 'supplied crawl review artifact consumed; live production crawl was not re-run' : 'report only consumes supplied crawl artifacts' }));
+
+ for (const finding of findings) {
+ if (finding.severity === 'warning' && finding.status === 'open') {
+ risks.push({ id: `risk-finding-${risks.length + 1}`, severity: finding.blocking ? 'high' : 'medium', status: 'open', title: finding.title, reason: finding.message, code: finding.code, subject: finding.subject });
+ }
+ }
+ for (const item of assurances.filter((entry) => entry.status === 'not-assessed')) {
+ risks.push({ id: `risk-${item.id}`, severity: 'medium', status: 'open', title: `${item.title} 未评估`, reason: item.evidence || '没有对应证据生产器。' });
+ }
+ for (const impact of impacts) {
+ risks.push({ id: `risk-derivation-${impact.name}`, severity: 'medium', status: 'open', title: `Derivation ${impact.name} 受到影响`, reason: (impact.triggers || []).join(', '), consumers: impact.consumers || [] });
+ }
+
+ if (outDir) {
+ artifacts.push({ id: 'report-json', type: 'output', path: safeRelative(path.join(outDir, 'report.json')), digest: null });
+ artifacts.push({ id: 'report-markdown', type: 'output', path: safeRelative(path.join(outDir, 'REPORT.md')), digest: null });
+ }
+ return buildReport({
+ command: 'report', libId, mode: reportMode(), pack, inputError, inputs,
+ findings, changes, assurances, coverage: assurances, risks, operations, impacts, artifacts, raw,
+ });
+}
+
+function writeReportOutputs(directory, report) {
+ fs.mkdirSync(directory, { recursive: true });
+ const nonce = `${process.pid}.${crypto.randomBytes(6).toString('hex')}`;
+ const outputs = [
+ { target: path.join(directory, 'report.json'), content: renderJson(report) },
+ { target: path.join(directory, 'REPORT.md'), content: renderMarkdown(report) },
+ ].map((entry) => ({ ...entry, temp: `${entry.target}.${nonce}.tmp`, backup: `${entry.target}.${nonce}.bak` }));
+ const installed = [];
+ const backedUp = [];
+ try {
+ for (const output of outputs) fs.writeFileSync(output.temp, output.content, { encoding: 'utf8', flag: 'wx' });
+ for (const output of outputs) {
+ if (fs.existsSync(output.target)) {
+ fs.renameSync(output.target, output.backup);
+ backedUp.push(output);
+ }
+ }
+ for (const output of outputs) {
+ fs.renameSync(output.temp, output.target);
+ installed.push(output);
+ }
+ for (const output of backedUp) {
+ try { fs.unlinkSync(output.backup); } catch (_) { /* a leftover backup is safer than losing the original */ }
+ }
+ } catch (error) {
+ for (const output of installed) {
+ try { fs.unlinkSync(output.target); } catch (_) { /* not installed */ }
+ }
+ for (const output of backedUp.reverse()) {
+ try { fs.renameSync(output.backup, output.target); } catch (_) { /* preserve original error */ }
+ }
+ throw error;
+ } finally {
+ for (const output of outputs) {
+ try { fs.unlinkSync(output.temp); } catch (_) { /* renamed or never created */ }
+ try { fs.unlinkSync(output.backup); } catch (_) { /* restored or never created */ }
+ }
+ }
+}
+
+let report;
+let outputWriteError = false;
+try {
+ report = collectReport();
+} catch (error) {
+ report = buildReport({
+ command: 'report', libId: path.basename(libDir), mode: reportMode(), inputError: true,
+ findings: [diagnosticFinding({ value: error, phase: 'report', category: 'contract', severity: 'error', blocking: true, code: 'REPORT_INPUT_ERROR', title: '报告生成失败' })],
+ assurances: [], inputs: { strict, verifyHash },
+ });
+}
+
+if (outDir && report.run.outcome !== 'input-error') {
+ try {
+ writeReportOutputs(outDir, report);
+ } catch (error) {
+ outputWriteError = true;
+ console.error('report: could not write report outputs: ' + error.message);
+ }
+}
+
+const reportPath = outDir && report.run.outcome !== 'input-error' ? safeRelative(path.join(outDir, 'REPORT.md')) : null;
+if (asJson) process.stdout.write(renderJson(report));
+else process.stdout.write(renderTerminalSummary(report, { reportPath }));
+process.exitCode = outputWriteError ? 2 : report.run.exitCode;
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-rules.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-rules.js
new file mode 100644
index 0000000..11863fe
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-rules.js
@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-rules.js — library-level data-rules executor
+ *
+ * Usage:
+ * node sg-pack-rules.js [--strict] [--rule ]
+ */
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { executeRules } = require('./lib/sg-pack-rules-core.js');
+
+const usage = 'Usage: node sg-pack-rules.js [--strict] [--rule ]';
+
+function parseArgs(argv) {
+ let input = null;
+ let strict = false;
+ let ruleId = null;
+ const seen = new Set();
+ for (let i = 0; i < argv.length; i += 1) {
+ const arg = argv[i];
+ if (arg === '--strict') {
+ if (seen.has(arg)) throw new Error('duplicate option: --strict');
+ seen.add(arg);
+ strict = true;
+ continue;
+ }
+ if (arg === '--rule') {
+ if (seen.has(arg)) throw new Error('duplicate option: --rule');
+ seen.add(arg);
+ if (i + 1 >= argv.length || argv[i + 1].startsWith('--')) throw new Error('--rule requires an id');
+ ruleId = argv[++i];
+ continue;
+ }
+ if (arg.startsWith('-')) throw new Error('unknown option: ' + arg);
+ if (input !== null) throw new Error('unexpected positional argument: ' + arg);
+ input = arg;
+ }
+ if (!input) throw new Error('library or rules path is required');
+ return { input, strict, ruleId };
+}
+
+let options;
+try {
+ options = parseArgs(process.argv.slice(2));
+} catch (error) {
+ console.error(error.message);
+ console.error(usage);
+ process.exit(2);
+}
+
+let rulesFile;
+let dataFile;
+if (fs.existsSync(options.input) && fs.statSync(options.input).isDirectory()) {
+ rulesFile = path.join(options.input, 'lib', 'data', 'data-rules.json');
+ dataFile = path.join(options.input, 'lib', 'data', 'data.json');
+} else {
+ rulesFile = path.resolve(process.cwd(), options.input);
+ dataFile = path.join(path.dirname(rulesFile), 'data.json');
+}
+for (const file of [rulesFile, dataFile]) {
+ if (!fs.existsSync(file)) {
+ console.error('File not found: ' + file);
+ process.exit(2);
+ }
+}
+
+let rulesDoc;
+let pack;
+try { rulesDoc = JSON.parse(fs.readFileSync(rulesFile, 'utf8')); }
+catch (error) { console.error('data-rules.json parse failed: ' + error.message); process.exit(1); }
+try { pack = JSON.parse(fs.readFileSync(dataFile, 'utf8')); }
+catch (error) { console.error('data.json parse failed: ' + error.message); process.exit(1); }
+
+const ruleIds = options.ruleId === null ? undefined : [options.ruleId];
+const result = executeRules(rulesDoc, pack, { ruleIds });
+if (result.structuralErrors.length) {
+ result.structuralErrors.forEach((error) => console.error('[STRUCT] ' + error));
+ console.error('✖ Rules-file structural check failed: ' + result.structuralErrors.length + ' error(s)');
+ process.exit(1);
+}
+if (result.selectionErrors && result.selectionErrors.length) {
+ result.selectionErrors.forEach((error) => console.error('[SELECT] ' + error));
+ process.exit(2);
+}
+if (result.unknownRuleIds.length) {
+ console.error('Unknown rule id: ' + result.unknownRuleIds.join(', '));
+ process.exit(2);
+}
+
+for (const item of result.results) {
+ if (item.status === 'passed' || item.status === 'no-check') continue;
+ if (item.status === 'error') {
+ console.error('[ERROR] ' + item.id + ' (' + item.level + '): check threw — ' + item.exception.message);
+ continue;
+ }
+ const hint = (item.subject ? '\n → subject: ' + item.subject : '') +
+ (item.repairHint ? '\n → repair: ' + item.repairHint : '');
+ if (item.level === 'hard') console.error('[FAIL] ' + item.id + ' (hard): ' + item.rule + hint);
+ else console.warn('[softfail] ' + item.id + ' (soft): ' + item.rule + hint);
+}
+
+const counts = result.counts;
+console.log('── ' + result.libId + ' ──────────────────');
+console.log('rules: ' + counts.total + ' | passed: ' + counts.passed + ' | hard-fail: ' + counts.hardFail + ' | soft-fail: ' + counts.softFail + ' | exec-error: ' + counts.errors + ' | no-check: ' + counts.noCheck);
+if (counts.hardFail || counts.errors) {
+ console.error('✖ Rules check failed');
+ process.exit(1);
+}
+if (options.strict && counts.softFail) {
+ console.error('✖ Strict mode: ' + counts.softFail + ' soft failure(s)');
+ process.exit(1);
+}
+console.log('✔ Rules check passed');
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-task.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-task.js
new file mode 100644
index 0000000..0ae47a6
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-task.js
@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+'use strict';
+const fs = require('node:fs');
+const path = require('node:path');
+const { validateTaskManifest, readTaskManifest } = require('./lib/agent-task-manifest.js');
+const { runTask } = require('./lib/sg-task-runner.js');
+
+const [, , action, taskArg, ...args] = process.argv;
+const usage = [
+ 'Usage:',
+ ' sg-data-pack task validate [--shape-only] [--json]',
+ ' sg-data-pack task inspect [--json]',
+ ' sg-data-pack task run --agent-command [--agent-arg ]... --artifacts [--provider name] [--model name] [--json]',
+].join('\n');
+if (!action || !taskArg || !['validate', 'inspect', 'run'].includes(action)) { console.error(usage); process.exit(2); }
+
+function parse(allowedValue, allowedBoolean, repeatable = new Set()) {
+ const options = {};
+ for (let index = 0; index < args.length; index += 1) {
+ const name = args[index];
+ if (allowedBoolean.has(name)) {
+ if (Object.prototype.hasOwnProperty.call(options, name)) throw new Error(`Duplicate option: ${name}`);
+ options[name] = true;
+ continue;
+ }
+ if (!allowedValue.has(name)) throw new Error(`Unknown option: ${name}`);
+ const value = args[index + 1];
+ if (!value || value.startsWith('--')) throw new Error(`${name} requires a value`);
+ if (repeatable.has(name)) {
+ if (!options[name]) options[name] = [];
+ options[name].push(value);
+ } else {
+ if (Object.prototype.hasOwnProperty.call(options, name)) throw new Error(`Duplicate option: ${name}`);
+ options[name] = value;
+ }
+ index += 1;
+ }
+ return options;
+}
+
+let options;
+try {
+ options = action === 'run'
+ ? parse(new Set(['--agent-command', '--agent-arg', '--artifacts', '--provider', '--model']), new Set(['--json', '--keep-workspace']), new Set(['--agent-arg']))
+ : parse(new Set(), new Set(['--json', '--shape-only']));
+} catch (error) { console.error(error.message); process.exit(2); }
+const taskFile = path.resolve(taskArg);
+
+try {
+ if (action === 'validate' || action === 'inspect') {
+ const manifest = JSON.parse(fs.readFileSync(taskFile, 'utf8'));
+ const validation = validateTaskManifest(manifest, {
+ taskFile,
+ verifyFiles: !options['--shape-only'],
+ verifyTree: !options['--shape-only'],
+ });
+ if (options['--json']) process.stdout.write(JSON.stringify({ manifest: action === 'inspect' ? manifest : undefined, validation }, null, 2) + '\n');
+ else {
+ console.log(`Agent task ${manifest.taskId || ''}: ${validation.valid ? 'valid' : 'invalid'}`);
+ console.log(`source: ${validation.sourceRoot || 'not resolved'}`);
+ validation.issues.forEach((item) => console.error(` ${item.path}: ${item.message}`));
+ }
+ process.exit(validation.valid ? 0 : 2);
+ }
+ if (!options['--agent-command'] || !options['--artifacts']) throw new Error('task run requires --agent-command and --artifacts');
+ readTaskManifest(taskFile);
+ const agentArgv = [options['--agent-command'], ...(options['--agent-arg'] || [])];
+ const result = runTask({
+ taskFile,
+ agentArgv,
+ artifactRoot: path.resolve(options['--artifacts']),
+ agent: { provider: options['--provider'] || null, model: options['--model'] || null },
+ keepWorkspace: Boolean(options['--keep-workspace']),
+ });
+ if (options['--json']) process.stdout.write(JSON.stringify(result.report, null, 2) + '\n');
+ else {
+ console.log(`TaskRun ${result.report.runId}`);
+ console.log(`verdict: ${result.report.verdict}`);
+ console.log(`patch: ${result.report.patch ? result.report.patch.status : 'not produced'}`);
+ result.report.grades.forEach((grade) => console.log(`grade ${grade.graderId}: ${grade.verdict} (${grade.score}/${grade.maximumScore})`));
+ console.log(`report: ${result.runFile}`);
+ }
+ process.exit(result.report.termination.processExitCode);
+} catch (error) {
+ console.error(`task ${action}: ${error.message}`);
+ process.exit(error.kind === 'input-error' || error.code && /MANIFEST/.test(error.code) ? 2 : 3);
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-templatize.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-templatize.js
new file mode 100644
index 0000000..70ee223
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-templatize.js
@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-templatize.js — auto-derive an item template from N repeated HTML
+ * instances via multi-way token alignment (LCS diff of instance[0] vs each other).
+ *
+ * Generalized from the diegovz stage-2 parameterizer. Use it for collection-type
+ * pages (card lists, timelines, leaderboards): the output is a byte-exact
+ * template + per-instance slot values, ready to be mapped to semantic fields.
+ *
+ * Usage:
+ * sg-data-pack templatize [--out ] [--prefix ]
+ *
+ * Input JSON: an array of HTML strings (the repeated item instances, in order),
+ * e.g. extracted from the page with any DOM tool you like.
+ *
+ * Output (with --out, otherwise report only):
+ * -template.txt HTML template with @@FIELD:f@@ slots
+ * -values.json { slotCount, instances: [[slotValue, ...] x N] }
+ *
+ * Guarantees: fill(template, values[i]) === instances[i] for every i (byte-exact),
+ * verified before exit. Exit 1 if verification fails.
+ *
+ * Next steps after running (manual, see references/data-rules-guide.md):
+ * 1. name the slots semantically (title/period/url/eid...) from the value samples
+ * 2. write the item renderer (template + slot builders) into the library
+ * 3. extract slot values into Data Pack entities
+ */
+'use strict';
+const fs = require('fs');
+const path = require('path');
+
+const [, , input, ...rest] = process.argv;
+if (!input) {
+ console.error('Usage: sg-data-pack templatize [--out ] [--prefix ]');
+ process.exit(2);
+}
+const flag = (n) => { const i = rest.indexOf(n); return i >= 0 ? rest[i + 1] : null; };
+const outDir = flag('--out');
+const prefix = flag('--prefix') || 'item';
+
+let instances;
+try {
+ instances = JSON.parse(fs.readFileSync(input, 'utf8'));
+} catch (e) {
+ console.error(`Cannot read ${input}: ${e.message}`);
+ process.exit(2);
+}
+if (!Array.isArray(instances) || instances.length < 2 || instances.some((x) => typeof x !== 'string')) {
+ console.error('Input must be a JSON array of >=2 HTML strings.');
+ process.exit(2);
+}
+
+/* ---------- tokenize: tags | newline+indent | text runs (lossless join) ---------- */
+function tokenize(s) {
+ return s.match(/<[^>]*>|\n[^\S\n]*|[^<\n]+|\n/g) || [];
+}
+
+/* ---------- LCS alignment ---------- */
+function align(base, inst) {
+ const n = base.length, m = inst.length;
+ const dp = new Uint32Array((n + 1) * (m + 1));
+ const W = m + 1;
+ for (let i = n - 1; i >= 0; i--) {
+ for (let j = m - 1; j >= 0; j--) {
+ dp[i * W + j] = base[i] === inst[j]
+ ? dp[(i + 1) * W + j + 1] + 1
+ : Math.max(dp[(i + 1) * W + j], dp[i * W + j + 1]);
+ }
+ }
+ const matched = new Map();
+ let i = 0, j = 0;
+ while (i < n && j < m) {
+ if (base[i] === inst[j]) { matched.set(i, j); i++; j++; }
+ else if (dp[(i + 1) * W + j] >= dp[i * W + j + 1]) i++;
+ else j++;
+ }
+ return { matched, instLen: m };
+}
+
+/* ---------- derive template across all instances ---------- */
+function deriveTemplate(instances) {
+ const base = tokenize(instances[0]);
+ const others = instances.slice(1).map(tokenize);
+ const aligns = others.map((t) => align(base, t));
+ const n = base.length;
+
+ // stable[i]: matched in every instance AND no insertion immediately before it
+ const stable = new Array(n).fill(true);
+ for (const { matched } of aligns) {
+ for (let i = 0; i < n; i++) if (!matched.has(i)) stable[i] = false;
+ let prevI = -1;
+ for (const [b, ii] of [...matched.entries()].sort((a, b2) => a[0] - b2[0])) {
+ if (prevI !== -1 && ii !== prevI + 1) stable[b] = false; // insertion before b
+ prevI = ii;
+ }
+ }
+
+ const segments = [];
+ let cur = null;
+ for (let i = 0; i < n; i++) {
+ if (stable[i]) {
+ if (!cur || cur.type !== 'static') { cur = { type: 'static', text: '' }; segments.push(cur); }
+ cur.text += base[i];
+ } else {
+ if (!cur || cur.type !== 'slot') { cur = { type: 'slot', id: segments.filter((s) => s.type === 'slot').length, baseStart: i, baseEnd: i }; segments.push(cur); }
+ cur.baseEnd = i + 1;
+ }
+ }
+
+ // per-instance slot values — boundaries via STATIC tokens only
+ const perInstance = [];
+ const allTok = [base, ...others];
+ const allAlign = [{ matched: new Map(base.map((_, i) => [i, i])), instLen: n }, ...aligns];
+ for (let k = 0; k < instances.length; k++) {
+ const { matched, instLen } = allAlign[k];
+ const tok = allTok[k];
+ const values = [];
+ for (const seg of segments) {
+ if (seg.type !== 'slot') continue;
+ let start = 0;
+ for (let b = seg.baseStart - 1; b >= 0; b--) if (stable[b] && matched.has(b)) { start = matched.get(b) + 1; break; }
+ let end = instLen;
+ for (let b = seg.baseEnd; b < n; b++) if (stable[b] && matched.has(b)) { end = matched.get(b); break; }
+ values[seg.id] = tok.slice(start, end).join('');
+ }
+ perInstance.push(values);
+ }
+
+ const slotCount = segments.filter((s) => s.type === 'slot').length;
+ const template = segments.map((s) => (s.type === 'static' ? s.text : `@@FIELD:f${s.id}@@`)).join('');
+ return { template, slotCount, perInstance };
+}
+
+function fillTemplate(template, values) {
+ return template.replace(/@@FIELD:f(\d+)@@/g, (m, i) => values[+i]);
+}
+
+/* ---------- run ---------- */
+const { template, slotCount, perInstance } = deriveTemplate(instances);
+
+let ok = true;
+for (let i = 0; i < instances.length; i++) {
+ if (fillTemplate(template, perInstance[i]) !== instances[i]) { ok = false; console.error(`verify FAIL at instance ${i}`); break; }
+}
+
+const staticBytes = template.replace(/@@FIELD:f\d+@@/g, '').length;
+const slotBytes = perInstance.flat().join('').length;
+console.log(`instances: ${instances.length}`);
+console.log(`slots: ${slotCount}`);
+console.log(`template: ${template.length}b (static ${staticBytes}b), values: ${slotBytes}b`);
+console.log(`byte-exact verification: ${ok ? 'OK' : 'FAILED'}`);
+if (!ok) process.exit(1);
+
+// slot value samples to aid semantic naming
+console.log('\nslot samples (up to 2 distinct values each):');
+for (let s = 0; s < slotCount; s++) {
+ const uniq = [...new Set(perInstance.map((v) => v[s]))];
+ const samples = uniq.slice(0, 2).map((x) => (x.length > 70 ? x.slice(0, 70) + `…(${x.length}b)` : x));
+ console.log(` f${s} (${uniq.length} distinct): ${JSON.stringify(samples)}`);
+}
+
+if (outDir) {
+ fs.mkdirSync(outDir, { recursive: true });
+ fs.writeFileSync(path.join(outDir, `${prefix}-template.txt`), template);
+ fs.writeFileSync(path.join(outDir, `${prefix}-values.json`), JSON.stringify({ slotCount, instances: perInstance }, null, 2));
+ console.log(`\nwrote ${prefix}-template.txt + ${prefix}-values.json to ${outDir}`);
+}
diff --git a/research/agent-eval/results/executed-tooling/scripts/sg-pack-types.js b/research/agent-eval/results/executed-tooling/scripts/sg-pack-types.js
new file mode 100644
index 0000000..1001589
--- /dev/null
+++ b/research/agent-eval/results/executed-tooling/scripts/sg-pack-types.js
@@ -0,0 +1,196 @@
+#!/usr/bin/env node
+/*
+ * sg-pack-types.js — generate TypeScript declarations from a Data Pack.
+ *
+ * Two layers:
+ * 1. contract layer: DataPack base interfaces (from the universal contract)
+ * 2. library layer: per-kind entity interfaces inferred from the actual
+ * data.json (field unions, optional markers, literal enums, id unions)
+ *
+ * Usage:
+ * sg-data-pack types [--out lib/src/data-types.d.ts] [--name ]
+ *
+ * Default is derived from meta.id (diegovz-home-0722-ts -> DiegovzHome0722Ts
+ * is ugly, so prefer passing --name, e.g. --name Diegovz).
+ *
+ * Engines can then opt into checking WITHOUT a build step:
+ * // @ts-check
+ * /** @type {import('./data-types').DiegovzPack} *\/ (pack)
+ *
+ * Inference rules:
+ * - entities grouped by kind; fields = union across the group
+ * - field optional (?) when absent in any member of the group
+ * - string fields with <=8 distinct short values and full coverage -> literal union
+ * - arrays: homogeneous element literal union where possible, else unknown[]
+ * - nested objects: inline interface when consistent shape, else Record
+ */
+'use strict';
+const fs = require('fs');
+const path = require('path');
+
+const [, , packPath, ...rest] = process.argv;
+if (!packPath) {
+ console.error('Usage: sg-data-pack types [--out ] [--name ]');
+ process.exit(2);
+}
+const flag = (n) => {
+ const i = rest.indexOf(n);
+ if (i < 0) return null;
+ const value = rest[i + 1];
+ if (!value || value.startsWith('--')) {
+ console.error(`${n} requires a value`);
+ process.exit(2);
+ }
+ return value;
+};
+const pack = JSON.parse(fs.readFileSync(packPath, 'utf8'));
+const explicitName = flag('--name');
+if (explicitName && !/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(explicitName)) {
+ console.error(`--name must be a valid TypeScript identifier, got ${JSON.stringify(explicitName)}`);
+ process.exit(2);
+}
+const libName = explicitName || (() => {
+ const id = (pack.meta && pack.meta.id) || 'Library';
+ const derived = id.replace(/(^|[-_])(\w)/g, (m, p, c) => c.toUpperCase()).replace(/[^A-Za-z0-9_$]/g, '');
+ if (!derived) return 'Library';
+ return /^[A-Za-z_$]/.test(derived) ? derived : `Pack${derived}`;
+})();
+const outPath = flag('--out');
+
+const ENUM_MAX = 8;
+
+/* ---------- type inference ---------- */
+function pascal(s) {
+ // split on - _ and any non-alphanumeric, capitalize each part, join
+ return String(s).split(/[^A-Za-z0-9]+/).filter(Boolean).map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join('');
+}
+function tsLit(s) { return JSON.stringify(s); }
+
+function inferValueType(values, depth) {
+ const types = new Set(values.map((v) => (Array.isArray(v) ? 'array' : v === null ? 'null' : typeof v)));
+ if (types.size > 1) return 'unknown';
+ const t = [...types][0];
+ if (t === 'string') {
+ const uniq = [...new Set(values)];
+ if (uniq.length <= ENUM_MAX && uniq.every((s) => s.length <= 40)) {
+ return uniq.map(tsLit).join(' | ');
+ }
+ return 'string';
+ }
+ if (t === 'number') return 'number';
+ if (t === 'boolean') return 'boolean';
+ if (t === 'array') {
+ const elems = values.flat();
+ if (!elems.length) return 'unknown[]';
+ const et = inferValueType(elems, depth + 1);
+ return et.includes('\n') ? 'unknown[]' : `(${et})[]`.replace(/^\((string|number|boolean)\)\[\]$/, '$1[]');
+ }
+ if (t === 'object' && depth === 0) {
+ // inline interface if all members share a consistent key set
+ const keySets = values.map((v) => Object.keys(v).sort().join(''));
+ if (new Set(keySets).size === 1) {
+ const keys = Object.keys(values[0]);
+ const fields = keys.map((k) => ` ${JSON.stringify(k)}: ${inferValueType(values.map((v) => v[k]), depth + 1)};`).join('\n');
+ return `{\n${fields}\n }`;
+ }
+ return 'Record';
+ }
+ return 'unknown';
+}
+
+/* ---------- entity inference ---------- */
+const byKind = new Map();
+for (const [id, e] of Object.entries(pack.entities || {})) {
+ const k = e.kind || 'unknown';
+ if (!byKind.has(k)) byKind.set(k, []);
+ byKind.get(k).push([id, e]);
+}
+
+const kindInterfaces = [];
+for (const [kind, members] of [...byKind.entries()].sort()) {
+ const iname = `${libName}${pascal(kind)}Entity`;
+ const fields = new Map(); // field -> {values, coverage}
+ for (const [, e] of members) {
+ for (const [f, v] of Object.entries(e)) {
+ if (f === 'kind') continue;
+ if (!fields.has(f)) fields.set(f, []);
+ fields.get(f).push(v);
+ }
+ }
+ const lines = [`export interface ${iname} {`, ` kind: ${tsLit(kind)};`];
+ for (const [f, vals] of [...fields.entries()].sort()) {
+ const optional = vals.length < members.length ? '?' : '';
+ lines.push(` ${JSON.stringify(f)}${optional}: ${inferValueType(vals, 0)};`);
+ }
+ lines.push('}');
+ kindInterfaces.push({ kind, iname, code: lines.join('\n') });
+}
+
+const entityUnion = kindInterfaces.map((k) => k.iname).join(' | ') || 'never';
+const idUnion = Object.keys(pack.entities || {}).map(tsLit).join(' | ') || 'never';
+
+/* ---------- emit ---------- */
+const out = `/* Auto-generated from data.json by sg-data-pack types (${libName}).
+ * Do not edit — regenerate: sg-data-pack types lib/data/data.json --name ${libName} */
+
+/* ---------- contract layer (universal Data Pack sections) ---------- */
+export interface SgRelation { id?: string; a: string; b: string; type: string; label?: string; scope?: string[]; }
+export interface SgProvenanceEntry {
+ origin?: string;
+ sourceUrl?: string | null;
+ fetchedAt?: string;
+ confidence?: number;
+ note?: string;
+ fieldOrigins?: Record;
+ [k: string]: unknown;
+}
+export interface SgAssetEntry { exists?: boolean; bytes?: number; hash?: string; sourceUrl?: string; }
+export interface SgDerivation {
+ kind: 'repeat' | 'insertion-order' | 'lookup-rebuild' | 'scope-resolution' | 'projection' | 'reference-only';
+ source: string;
+ consumers: string[];
+ affects?: string[];
+ alsoTouches?: string[];
+ note: string;
+}
+
+/* ---------- library layer (inferred from data) ---------- */
+${kindInterfaces.map((k) => k.code).join('\n\n')}
+
+export type ${libName}Entity = ${entityUnion};
+export type ${libName}EntityId = ${idUnion};
+
+export interface ${libName}Pack {
+ schemaVersion: '1.0' | '1.1' | '1.2' | '1.3';
+ meta: { id: string; title: string; [k: string]: unknown };
+ entities: Record;
+ aliases?: Record;
+ relationTypes?: Record;
+ heroRelTypes?: Record;
+ relations?: SgRelation[];
+ stages?: Array>;
+ contents?: Record>;
+ domain?: Record;
+ assets?: Record;
+ attributeTypes?: Record;
+ attributeSources?: string[];
+ kindNameFields?: Record;
+ sameAs?: Array<[string, string]>;
+ derivations?: Record;
+ provenance?: {
+ entities?: Record;
+ relations?: Record;
+ contents?: Record