From 370a6786eafb622726815bd80e95cad02458b32f Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 22 Jul 2026 15:07:50 +0100 Subject: [PATCH 1/2] feat(harness): prompt eval runner benching system-prompt variants on a live rig --- harness/evals/prompts/.gitignore | 1 + harness/evals/prompts/README.md | 79 ++++++++++ harness/evals/prompts/run.mjs | 222 +++++++++++++++++++++++++++ harness/evals/prompts/scenarios.json | 78 ++++++++++ 4 files changed, 380 insertions(+) create mode 100644 harness/evals/prompts/.gitignore create mode 100644 harness/evals/prompts/README.md create mode 100644 harness/evals/prompts/run.mjs create mode 100644 harness/evals/prompts/scenarios.json diff --git a/harness/evals/prompts/.gitignore b/harness/evals/prompts/.gitignore new file mode 100644 index 00000000..89f9ac04 --- /dev/null +++ b/harness/evals/prompts/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/harness/evals/prompts/README.md b/harness/evals/prompts/README.md new file mode 100644 index 00000000..038f4354 --- /dev/null +++ b/harness/evals/prompts/README.md @@ -0,0 +1,79 @@ +# Prompt evals + +Benches system-prompt variants against a live rig and grades agent conduct, not +exact text: what the agent called, in what order, what it avoided, and what the +final result contained. Use it before changing any identity prompt, playbook +skill, or injected guidance. + +## What it is (and is not) + +- `harness/evals/integration` is the deterministic conformance suite: isolated + stack, scripted `router::*`, exact invariants, CI gate. A scripted router + cannot measure prompt-driven behavior, which is exactly what changes when a + prompt changes. +- This runner is the complement: live engine, live provider, live prompts. It + answers "does the smaller prompt still produce correct conduct" with a task + matrix instead of a diff review. +- The HarnessBench tech spec describes the full product (matrix runs, metrics + store, console view). This is its working seed: one file, zero dependencies, + markdown + JSON reports. + +## Requirements + +A running engine with harness, session-manager, context-manager, a real +provider, and the `iii` CLI on PATH. The fp worker should be connected if you +run the bulk scenario (it grades pipe conduct). + +## Usage + +``` +node run.mjs # all arms x all scenarios +node run.mjs --scenario bulk-two-fields # one scenario, both arms +node run.mjs --arm candidate # one arm +node run.mjs --address # non-default engine +``` + +Exit code 0 when every non-optional assertion passes; reports land in `out/` +(gitignored) as `report-.md` and `report-.json`, including each +arm's measured prompt tokens (`context::count-tokens`) and the full call +sequence per session. + +## Arms + +`scenarios.json` defines the arms: + +- `current` sends no override, so the session runs whatever the router serves + the rig today (provider identity or operator override). Its token count is + fetched live from `router::system_prompt::get`. +- `candidate` reads `harness/prompts/default.txt` from this checkout and sends + it with `system_prompt_strategy: override`. + +Point `system_prompt_file` at any file to bench a different variant; add more +arms for a multi-way comparison. + +## Scenarios and assertions + +Each scenario is a user message plus assertions: + +- `result_matches` - regex over the final turn result. +- `calls_include` / `calls_exclude` - regex over the invoked function ids + (transcript order, including function results). +- `call_order` - the first match of `before` must precede the first match of + `after`; passes when `after` never fires. +- `arms` scopes an assertion to specific arms; `optional: true` records the + outcome without failing the run. + +The shipped matrix covers the conduct that prompt changes have actually +regressed or nearly regressed: no orchestration machinery on simple tasks, +playbook pull before spawning, bulk payloads moved with `fp::pipe` instead of +flooding the context (a 1 MB fetch read inline costs roughly 270k tokens), and +no installs on a read-only registry question. + +## Notes + +- Sessions are named `pbench---`; they stay on the rig + afterwards and are inspectable in the console. +- Every run sends `functions: { allow: ["*"] }`: a parentless `harness::send` + denies all dispatch by default, which would fail every scenario. +- Fan-out scenarios settle on a quiet window (no active turn for `quiet_s` + seconds) because reactions can wake a session after its first turn completes. diff --git a/harness/evals/prompts/run.mjs b/harness/evals/prompts/run.mjs new file mode 100644 index 00000000..f2ee3c93 --- /dev/null +++ b/harness/evals/prompts/run.mjs @@ -0,0 +1,222 @@ +#!/usr/bin/env node +// Prompt eval runner: benches system-prompt variants against a LIVE rig. +// +// For every (arm x scenario) pair it starts a fresh harness session via +// `iii trigger harness::send`, waits for the run to settle, reads the +// transcript back through `session::messages`, and grades it with the +// scenario's assertions. Arms differ only in the `system_prompt` / +// `system_prompt_strategy` send options; the "current" arm sends no +// override, so it exercises whatever the router serves the rig today. +// +// Requires: a running engine + harness + a real provider, and the `iii` +// CLI on PATH. This is deliberately NOT the deterministic conformance +// suite (harness/evals/integration): scripted routers cannot measure +// prompt-driven behavior, so these scenarios run against live models and +// grade conduct (what was called, in what order) rather than exact text. +// +// Usage: +// node run.mjs # all arms, all scenarios +// node run.mjs --scenario bulk-two-fields --arm candidate +// node run.mjs --address --out ./out + +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +function parseArgs(argv) { + const args = { out: join(HERE, "out") }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--scenario") args.scenario = argv[++i]; + else if (a === "--arm") args.arm = argv[++i]; + else if (a === "--address") args.address = argv[++i]; + else if (a === "--out") args.out = argv[++i]; + else if (a === "--help") { + console.log("node run.mjs [--scenario id[,id]] [--arm name[,name]] [--address host] [--out dir]"); + process.exit(0); + } + } + return args; +} + +const ARGS = parseArgs(process.argv.slice(2)); + +function trig(fn, payload, timeoutMs = 90_000) { + const cli = ["trigger", fn, "--json", JSON.stringify(payload)]; + if (ARGS.address) cli.push("--address", ARGS.address); + const r = spawnSync("iii", cli, { encoding: "utf8", timeout: timeoutMs }); + try { + return JSON.parse(r.stdout); + } catch { + return { _raw: (r.stdout || "").slice(-300), _err: (r.stderr || "").slice(-300) }; + } +} + +const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); + +function countTokens(model, text) { + const out = trig("context::count-tokens", { model: { id: model }, system_prompt: text, messages: [] }); + return out.tokens ?? out.token_count ?? null; +} + +async function waitSettled(sessionId, timeoutS, quietS) { + const t0 = Date.now(); + let quiet = 0; + while ((Date.now() - t0) / 1000 < timeoutS) { + await sleep(6000); + const st = trig("harness::status", { session_id: sessionId }); + const terminal = ["completed", "failed", "cancelled", undefined, null].includes(st.status); + quiet = terminal ? quiet + 6 : 0; + if (quiet >= quietS) break; + } + return trig("harness::status", { session_id: sessionId }); +} + +function readTranscript(sessionId) { + const resp = trig("session::messages", { session_id: sessionId }); + const calls = []; + let generations = 0; + for (const m of resp.messages ?? []) { + const msg = m.message ?? {}; + if (msg.role === "assistant") { + generations++; + for (const b of msg.content ?? []) { + if (b.type === "function_call") calls.push(b.arguments?.function ?? b.function_id ?? ""); + } + } else if (msg.role === "function_result") { + if (!calls.length || calls[calls.length - 1] !== msg.function_id) calls.push(msg.function_id); + } + } + return { calls: calls.filter(Boolean), generations }; +} + +function grade(assertions, armName, { calls, result }) { + const outcomes = []; + for (const a of assertions) { + if (a.arms && !a.arms.includes(armName)) continue; + const re = a.pattern ? new RegExp(a.pattern) : null; + let pass; + switch (a.type) { + case "result_matches": + pass = re.test(result ?? ""); + break; + case "calls_include": + pass = calls.some((c) => re.test(c)); + break; + case "calls_exclude": + pass = !calls.some((c) => re.test(c)); + break; + case "call_order": { + const before = calls.findIndex((c) => new RegExp(a.before).test(c)); + const after = calls.findIndex((c) => new RegExp(a.after).test(c)); + pass = after === -1 || (before !== -1 && before < after); + break; + } + default: + pass = false; + } + outcomes.push({ ...a, pass: pass || Boolean(a.optional), raw_pass: pass }); + } + return outcomes; +} + +const spec = JSON.parse(readFileSync(join(HERE, "scenarios.json"), "utf8")); +const runId = Date.now().toString(36).slice(-5); +const wantScenarios = ARGS.scenario ? ARGS.scenario.split(",") : null; +const wantArms = ARGS.arm ? ARGS.arm.split(",") : null; + +const arms = spec.arms + .filter((a) => !wantArms || wantArms.includes(a.name)) + .map((a) => ({ + ...a, + prompt: a.system_prompt_file ? readFileSync(resolve(HERE, a.system_prompt_file), "utf8") : null, + })); +const scenarios = spec.scenarios.filter((s) => !wantScenarios || wantScenarios.includes(s.id)); +if (!arms.length || !scenarios.length) { + console.error("nothing selected: check --arm / --scenario against scenarios.json"); + process.exit(2); +} + +const report = { run_id: runId, model: spec.model, arms: {}, results: [] }; +for (const arm of arms) { + const served = arm.prompt ?? trig("router::system_prompt::get", {}).system_prompt ?? ""; + report.arms[arm.name] = { + prompt_tokens: served ? countTokens(spec.model, served) : null, + source: + arm.system_prompt_file ?? + (served ? "router-served" : "router-served (no default provider resolved; count unavailable)"), + }; +} + +let failed = 0; +for (const scenario of scenarios) { + for (const arm of arms) { + const sessionId = `pbench-${runId}-${arm.name}-${scenario.id}`; + const options = { max_turns: scenario.max_turns, functions: { allow: ["*"] } }; + if (arm.prompt) { + options.system_prompt = arm.prompt; + options.system_prompt_strategy = arm.strategy ?? "override"; + } + const sent = trig("harness::send", { + session_id: sessionId, + model: spec.model, + message: scenario.message, + options, + }); + if (!sent.accepted) { + console.error(`SEND FAILED ${arm.name}/${scenario.id}:`, JSON.stringify(sent).slice(0, 200)); + failed++; + continue; + } + console.log(`running ${arm.name}/${scenario.id} (${sessionId})`); + const status = await waitSettled(sessionId, scenario.timeout_s, scenario.quiet_s); + const transcript = readTranscript(sessionId); + const outcomes = grade(scenario.assertions, arm.name, { + calls: transcript.calls, + result: String(status.result ?? ""), + }); + const scenarioFailed = outcomes.some((o) => !o.pass); + if (scenarioFailed) failed++; + report.results.push({ + scenario: scenario.id, + arm: arm.name, + session_id: sessionId, + status: status.status, + turns: status.turn_count, + generations: transcript.generations, + calls: transcript.calls, + result_head: String(status.result ?? "").slice(0, 200), + assertions: outcomes, + pass: !scenarioFailed, + }); + console.log(` ${scenarioFailed ? "FAIL" : "pass"} turns=${status.turn_count} gens=${transcript.generations}`); + } +} + +mkdirSync(ARGS.out, { recursive: true }); +writeFileSync(join(ARGS.out, `report-${runId}.json`), JSON.stringify(report, null, 2)); + +const lines = [ + `# Prompt eval run ${runId}`, + "", + `Model: ${spec.model}`, + "", + "| arm | prompt tokens | source |", + "|---|---|---|", + ...Object.entries(report.arms).map(([n, a]) => `| ${n} | ${a.prompt_tokens ?? "?"} | ${a.source} |`), + "", + "| scenario | arm | pass | turns | gens | calls |", + "|---|---|---|---|---|---|", + ...report.results.map( + (r) => `| ${r.scenario} | ${r.arm} | ${r.pass ? "pass" : "FAIL"} | ${r.turns} | ${r.generations} | ${r.calls.length} |` + ), +]; +writeFileSync(join(ARGS.out, `report-${runId}.md`), lines.join("\n") + "\n"); +console.log(`\nreport: ${join(ARGS.out, `report-${runId}.md`)}`); +if (failed) { + console.error(`${failed} scenario run(s) failed`); + process.exit(1); +} diff --git a/harness/evals/prompts/scenarios.json b/harness/evals/prompts/scenarios.json new file mode 100644 index 00000000..f4b471a8 --- /dev/null +++ b/harness/evals/prompts/scenarios.json @@ -0,0 +1,78 @@ +{ + "model": "claude-sonnet-5", + "arms": [ + { "name": "current", "note": "whatever the router serves today (provider identity or operator override)" }, + { + "name": "candidate", + "system_prompt_file": "../../prompts/default.txt", + "strategy": "override", + "note": "the embedded fallback prompt from this checkout" + } + ], + "scenarios": [ + { + "id": "simple-ls", + "message": "List the files under /tmp and tell me how many there are.", + "max_turns": 8, + "timeout_s": 150, + "quiet_s": 12, + "assertions": [ + { "type": "result_matches", "pattern": "\\d" }, + { "type": "calls_exclude", "pattern": "harness::spawn|engine::register_trigger" }, + { "type": "calls_exclude", "pattern": "directory::skills::get" } + ] + }, + { + "id": "discovery-workers", + "message": "Which workers are connected to the engine right now? Just give me the names.", + "max_turns": 8, + "timeout_s": 150, + "quiet_s": 12, + "assertions": [ + { "type": "calls_include", "pattern": "engine::workers::list" }, + { "type": "calls_exclude", "pattern": "harness::spawn" } + ] + }, + { + "id": "fanout-state-report", + "message": "Spawn 3 sub-agents in parallel. Each writes one interesting fact about the number 7, 8, and 9 respectively to state. Report back to me with all three facts once they are all in.", + "max_turns": 16, + "timeout_s": 480, + "quiet_s": 90, + "assertions": [ + { "type": "calls_include", "pattern": "harness::spawn" }, + { "type": "result_matches", "pattern": "7[\\s\\S]*8[\\s\\S]*9" }, + { + "type": "call_order", + "before": "directory::skills::get", + "after": "harness::spawn", + "arms": ["candidate"], + "optional": true, + "note": "playbook pull should precede orchestration when it happens; engine-native in-turn spawns may legitimately skip it" + } + ] + }, + { + "id": "bulk-two-fields", + "message": "Fetch https://iii.dev/docs/llms-full.txt and tell me two things: its total size in characters, and the first heading line in it.", + "max_turns": 8, + "timeout_s": 300, + "quiet_s": 12, + "assertions": [ + { "type": "calls_include", "pattern": "fp::pipe" }, + { "type": "calls_exclude", "pattern": "^web::fetch$", "note": "a direct fetch floods ~270k tokens of body into the context" }, + { "type": "result_matches", "pattern": "1[,.]?000[,.]?948|Helpers" } + ] + }, + { + "id": "registry-lookup-no-install", + "message": "Do we have any worker available that can send email? Do not install anything, just tell me what exists.", + "max_turns": 8, + "timeout_s": 150, + "quiet_s": 12, + "assertions": [ + { "type": "calls_exclude", "pattern": "worker::add" } + ] + } + ] +} From 7a476dd2d25d7b769d521e137e517023e11f6c7e Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 22 Jul 2026 17:13:08 +0100 Subject: [PATCH 2/2] refactor(harness): single-channel call grading and minor eval runner cleanups --- harness/evals/prompts/README.md | 8 ++++---- harness/evals/prompts/run.mjs | 29 +++++++++++++++-------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/harness/evals/prompts/README.md b/harness/evals/prompts/README.md index 038f4354..6a095174 100644 --- a/harness/evals/prompts/README.md +++ b/harness/evals/prompts/README.md @@ -7,10 +7,10 @@ skill, or injected guidance. ## What it is (and is not) -- `harness/evals/integration` is the deterministic conformance suite: isolated - stack, scripted `router::*`, exact invariants, CI gate. A scripted router - cannot measure prompt-driven behavior, which is exactly what changes when a - prompt changes. +- The deterministic conformance suite (in progress as `harness/evals/integration`) + boots an isolated stack with a scripted `router::*` and grades exact + invariants as a CI gate. A scripted router cannot measure prompt-driven + behavior, which is exactly what changes when a prompt changes. - This runner is the complement: live engine, live provider, live prompts. It answers "does the smaller prompt still produce correct conduct" with a task matrix instead of a diff review. diff --git a/harness/evals/prompts/run.mjs b/harness/evals/prompts/run.mjs index f2ee3c93..eceb8d10 100644 --- a/harness/evals/prompts/run.mjs +++ b/harness/evals/prompts/run.mjs @@ -10,7 +10,7 @@ // // Requires: a running engine + harness + a real provider, and the `iii` // CLI on PATH. This is deliberately NOT the deterministic conformance -// suite (harness/evals/integration): scripted routers cannot measure +// suite (in progress as harness/evals/integration): scripted routers cannot measure // prompt-driven behavior, so these scenarios run against live models and // grade conduct (what was called, in what order) rather than exact text. // @@ -59,36 +59,35 @@ const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); function countTokens(model, text) { const out = trig("context::count-tokens", { model: { id: model }, system_prompt: text, messages: [] }); - return out.tokens ?? out.token_count ?? null; + return out.tokens ?? null; } async function waitSettled(sessionId, timeoutS, quietS) { const t0 = Date.now(); let quiet = 0; + let st = {}; while ((Date.now() - t0) / 1000 < timeoutS) { await sleep(6000); - const st = trig("harness::status", { session_id: sessionId }); + st = trig("harness::status", { session_id: sessionId }); const terminal = ["completed", "failed", "cancelled", undefined, null].includes(st.status); quiet = terminal ? quiet + 6 : 0; if (quiet >= quietS) break; } - return trig("harness::status", { session_id: sessionId }); + return st; } function readTranscript(sessionId) { + // `calls` comes from the function_result rows alone: every executed call + // yields exactly one result, in execution order. (A call also appears as + // an assistant function_call block; counting both channels double-counts + // parallel pairs and inflates the grading.) const resp = trig("session::messages", { session_id: sessionId }); const calls = []; let generations = 0; for (const m of resp.messages ?? []) { const msg = m.message ?? {}; - if (msg.role === "assistant") { - generations++; - for (const b of msg.content ?? []) { - if (b.type === "function_call") calls.push(b.arguments?.function ?? b.function_id ?? ""); - } - } else if (msg.role === "function_result") { - if (!calls.length || calls[calls.length - 1] !== msg.function_id) calls.push(msg.function_id); - } + if (msg.role === "assistant") generations++; + else if (msg.role === "function_result") calls.push(msg.function_id); } return { calls: calls.filter(Boolean), generations }; } @@ -110,8 +109,10 @@ function grade(assertions, armName, { calls, result }) { pass = !calls.some((c) => re.test(c)); break; case "call_order": { - const before = calls.findIndex((c) => new RegExp(a.before).test(c)); - const after = calls.findIndex((c) => new RegExp(a.after).test(c)); + const beforeRe = new RegExp(a.before); + const afterRe = new RegExp(a.after); + const before = calls.findIndex((c) => beforeRe.test(c)); + const after = calls.findIndex((c) => afterRe.test(c)); pass = after === -1 || (before !== -1 && before < after); break; }