(MOT-4168) harness: prompt eval runner for live-rig conduct benching

harness/evals/prompts/.gitignore

@@ -0,0 +1 @@
+out/

harness/evals/prompts/README.md

@@ -0,0 +1,79 @@
+# Prompt evals
+
+Benches system-prompt variants against a live rig and grades agent conduct, not
+exact text: what the agent called, in what order, what it avoided, and what the
+final result contained. Use it before changing any identity prompt, playbook
+skill, or injected guidance.
+
+## What it is (and is not)
+
+- The deterministic conformance suite (in progress as `harness/evals/integration`)
+  boots an isolated stack with a scripted `router::*` and grades exact
+  invariants as a CI gate. A scripted router cannot measure prompt-driven
+  behavior, which is exactly what changes when a prompt changes.
+- This runner is the complement: live engine, live provider, live prompts. It
+  answers "does the smaller prompt still produce correct conduct" with a task
+  matrix instead of a diff review.
+- The HarnessBench tech spec describes the full product (matrix runs, metrics
+  store, console view). This is its working seed: one file, zero dependencies,
+  markdown + JSON reports.
+
+## Requirements
+
+A running engine with harness, session-manager, context-manager, a real
+provider, and the `iii` CLI on PATH. The fp worker should be connected if you
+run the bulk scenario (it grades pipe conduct).
+
+## Usage
+
+```
+node run.mjs                                  # all arms x all scenarios
+node run.mjs --scenario bulk-two-fields       # one scenario, both arms
+node run.mjs --arm candidate                  # one arm
+node run.mjs --address <engine-host>          # non-default engine
+```
+
+Exit code 0 when every non-optional assertion passes; reports land in `out/`
+(gitignored) as `report-<runid>.md` and `report-<runid>.json`, including each
+arm's measured prompt tokens (`context::count-tokens`) and the full call
+sequence per session.
+
+## Arms
+
+`scenarios.json` defines the arms:
+
+- `current` sends no override, so the session runs whatever the router serves
+  the rig today (provider identity or operator override). Its token count is
+  fetched live from `router::system_prompt::get`.
+- `candidate` reads `harness/prompts/default.txt` from this checkout and sends
+  it with `system_prompt_strategy: override`.
+
+Point `system_prompt_file` at any file to bench a different variant; add more
+arms for a multi-way comparison.
+
+## Scenarios and assertions
+
+Each scenario is a user message plus assertions:
+
+- `result_matches` - regex over the final turn result.
+- `calls_include` / `calls_exclude` - regex over the invoked function ids
+  (transcript order, including function results).
+- `call_order` - the first match of `before` must precede the first match of
+  `after`; passes when `after` never fires.
+- `arms` scopes an assertion to specific arms; `optional: true` records the
+  outcome without failing the run.
+
+The shipped matrix covers the conduct that prompt changes have actually
+regressed or nearly regressed: no orchestration machinery on simple tasks,
+playbook pull before spawning, bulk payloads moved with `fp::pipe` instead of
+flooding the context (a 1 MB fetch read inline costs roughly 270k tokens), and
+no installs on a read-only registry question.
+
+## Notes
+
+- Sessions are named `pbench-<runid>-<arm>-<scenario>`; they stay on the rig
+  afterwards and are inspectable in the console.
+- Every run sends `functions: { allow: ["*"] }`: a parentless `harness::send`
+  denies all dispatch by default, which would fail every scenario.
+- Fan-out scenarios settle on a quiet window (no active turn for `quiet_s`
+  seconds) because reactions can wake a session after its first turn completes.

harness/evals/prompts/run.mjs

@@ -0,0 +1,223 @@
+#!/usr/bin/env node
+// Prompt eval runner: benches system-prompt variants against a LIVE rig.
+//
+// For every (arm x scenario) pair it starts a fresh harness session via
+// `iii trigger harness::send`, waits for the run to settle, reads the
+// transcript back through `session::messages`, and grades it with the
+// scenario's assertions. Arms differ only in the `system_prompt` /
+// `system_prompt_strategy` send options; the "current" arm sends no
+// override, so it exercises whatever the router serves the rig today.
+//
+// Requires: a running engine + harness + a real provider, and the `iii`
+// CLI on PATH. This is deliberately NOT the deterministic conformance
+// suite (in progress as harness/evals/integration): scripted routers cannot measure
+// prompt-driven behavior, so these scenarios run against live models and
+// grade conduct (what was called, in what order) rather than exact text.
+//
+// Usage:
+//   node run.mjs                         # all arms, all scenarios
+//   node run.mjs --scenario bulk-two-fields --arm candidate
+//   node run.mjs --address <engine-host> --out ./out
+
+import { spawnSync } from "node:child_process";
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+
+function parseArgs(argv) {
+  const args = { out: join(HERE, "out") };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--scenario") args.scenario = argv[++i];
+    else if (a === "--arm") args.arm = argv[++i];
+    else if (a === "--address") args.address = argv[++i];
+    else if (a === "--out") args.out = argv[++i];
+    else if (a === "--help") {
+      console.log("node run.mjs [--scenario id[,id]] [--arm name[,name]] [--address host] [--out dir]");
+      process.exit(0);
+    }
+  }
+  return args;
+}
+
+const ARGS = parseArgs(process.argv.slice(2));
+
+function trig(fn, payload, timeoutMs = 90_000) {
+  const cli = ["trigger", fn, "--json", JSON.stringify(payload)];
+  if (ARGS.address) cli.push("--address", ARGS.address);
+  const r = spawnSync("iii", cli, { encoding: "utf8", timeout: timeoutMs });
+  try {
+    return JSON.parse(r.stdout);
+  } catch {
+    return { _raw: (r.stdout || "").slice(-300), _err: (r.stderr || "").slice(-300) };
+  }
+}
+
+const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+
+function countTokens(model, text) {
+  const out = trig("context::count-tokens", { model: { id: model }, system_prompt: text, messages: [] });
+  return out.tokens ?? null;
+}
+
+async function waitSettled(sessionId, timeoutS, quietS) {
+  const t0 = Date.now();
+  let quiet = 0;
+  let st = {};
+  while ((Date.now() - t0) / 1000 < timeoutS) {
+    await sleep(6000);
+    st = trig("harness::status", { session_id: sessionId });
+    const terminal = ["completed", "failed", "cancelled", undefined, null].includes(st.status);
+    quiet = terminal ? quiet + 6 : 0;
+    if (quiet >= quietS) break;
+  }
+  return st;
+}
+
+function readTranscript(sessionId) {
+  // `calls` comes from the function_result rows alone: every executed call
+  // yields exactly one result, in execution order. (A call also appears as
+  // an assistant function_call block; counting both channels double-counts
+  // parallel pairs and inflates the grading.)
+  const resp = trig("session::messages", { session_id: sessionId });
+  const calls = [];
+  let generations = 0;
+  for (const m of resp.messages ?? []) {
+    const msg = m.message ?? {};
+    if (msg.role === "assistant") generations++;
+    else if (msg.role === "function_result") calls.push(msg.function_id);
+  }
+  return { calls: calls.filter(Boolean), generations };
+}
+
+function grade(assertions, armName, { calls, result }) {
+  const outcomes = [];
+  for (const a of assertions) {
+    if (a.arms && !a.arms.includes(armName)) continue;
+    const re = a.pattern ? new RegExp(a.pattern) : null;
+    let pass;
+    switch (a.type) {
+      case "result_matches":
+        pass = re.test(result ?? "");
+        break;
+      case "calls_include":
+        pass = calls.some((c) => re.test(c));
+        break;
+      case "calls_exclude":
+        pass = !calls.some((c) => re.test(c));
+        break;
+      case "call_order": {
+        const beforeRe = new RegExp(a.before);
+        const afterRe = new RegExp(a.after);
+        const before = calls.findIndex((c) => beforeRe.test(c));
+        const after = calls.findIndex((c) => afterRe.test(c));
+        pass = after === -1 || (before !== -1 && before < after);
+        break;
+      }
+      default:
+        pass = false;
+    }
+    outcomes.push({ ...a, pass: pass || Boolean(a.optional), raw_pass: pass });
+  }
+  return outcomes;
+}
+
+const spec = JSON.parse(readFileSync(join(HERE, "scenarios.json"), "utf8"));
+const runId = Date.now().toString(36).slice(-5);
+const wantScenarios = ARGS.scenario ? ARGS.scenario.split(",") : null;
+const wantArms = ARGS.arm ? ARGS.arm.split(",") : null;
+
+const arms = spec.arms
+  .filter((a) => !wantArms || wantArms.includes(a.name))
+  .map((a) => ({
+    ...a,
+    prompt: a.system_prompt_file ? readFileSync(resolve(HERE, a.system_prompt_file), "utf8") : null,
+  }));
+const scenarios = spec.scenarios.filter((s) => !wantScenarios || wantScenarios.includes(s.id));
+if (!arms.length || !scenarios.length) {
+  console.error("nothing selected: check --arm / --scenario against scenarios.json");
+  process.exit(2);
+}
+
+const report = { run_id: runId, model: spec.model, arms: {}, results: [] };
+for (const arm of arms) {
+  const served = arm.prompt ?? trig("router::system_prompt::get", {}).system_prompt ?? "";
+  report.arms[arm.name] = {
+    prompt_tokens: served ? countTokens(spec.model, served) : null,
+    source:
+      arm.system_prompt_file ??
+      (served ? "router-served" : "router-served (no default provider resolved; count unavailable)"),
+  };
+}
+
+let failed = 0;
+for (const scenario of scenarios) {
+  for (const arm of arms) {
+    const sessionId = `pbench-${runId}-${arm.name}-${scenario.id}`;
+    const options = { max_turns: scenario.max_turns, functions: { allow: ["*"] } };
+    if (arm.prompt) {
+      options.system_prompt = arm.prompt;
+      options.system_prompt_strategy = arm.strategy ?? "override";
+    }
+    const sent = trig("harness::send", {
+      session_id: sessionId,
+      model: spec.model,
+      message: scenario.message,
+      options,
+    });
+    if (!sent.accepted) {
+      console.error(`SEND FAILED ${arm.name}/${scenario.id}:`, JSON.stringify(sent).slice(0, 200));
+      failed++;
+      continue;
+    }
+    console.log(`running ${arm.name}/${scenario.id} (${sessionId})`);
+    const status = await waitSettled(sessionId, scenario.timeout_s, scenario.quiet_s);
+    const transcript = readTranscript(sessionId);
+    const outcomes = grade(scenario.assertions, arm.name, {
+      calls: transcript.calls,
+      result: String(status.result ?? ""),
+    });
+    const scenarioFailed = outcomes.some((o) => !o.pass);
+    if (scenarioFailed) failed++;
+    report.results.push({
+      scenario: scenario.id,
+      arm: arm.name,
+      session_id: sessionId,
+      status: status.status,
+      turns: status.turn_count,
+      generations: transcript.generations,
+      calls: transcript.calls,
+      result_head: String(status.result ?? "").slice(0, 200),
+      assertions: outcomes,
+      pass: !scenarioFailed,
+    });
+    console.log(`  ${scenarioFailed ? "FAIL" : "pass"} turns=${status.turn_count} gens=${transcript.generations}`);
+  }
+}
+
+mkdirSync(ARGS.out, { recursive: true });
+writeFileSync(join(ARGS.out, `report-${runId}.json`), JSON.stringify(report, null, 2));
+
+const lines = [
+  `# Prompt eval run ${runId}`,
+  "",
+  `Model: ${spec.model}`,
+  "",
+  "| arm | prompt tokens | source |",
+  "|---|---|---|",
+  ...Object.entries(report.arms).map(([n, a]) => `| ${n} | ${a.prompt_tokens ?? "?"} | ${a.source} |`),
+  "",
+  "| scenario | arm | pass | turns | gens | calls |",
+  "|---|---|---|---|---|---|",
+  ...report.results.map(
+    (r) => `| ${r.scenario} | ${r.arm} | ${r.pass ? "pass" : "FAIL"} | ${r.turns} | ${r.generations} | ${r.calls.length} |`
+  ),
+];
+writeFileSync(join(ARGS.out, `report-${runId}.md`), lines.join("\n") + "\n");
+console.log(`\nreport: ${join(ARGS.out, `report-${runId}.md`)}`);
+if (failed) {
+  console.error(`${failed} scenario run(s) failed`);
+  process.exit(1);
+}

harness/evals/prompts/scenarios.json

@@ -0,0 +1,78 @@
+{
+  "model": "claude-sonnet-5",
+  "arms": [
+    { "name": "current", "note": "whatever the router serves today (provider identity or operator override)" },
+    {
+      "name": "candidate",
+      "system_prompt_file": "../../prompts/default.txt",
+      "strategy": "override",
+      "note": "the embedded fallback prompt from this checkout"
+    }
+  ],
+  "scenarios": [
+    {
+      "id": "simple-ls",
+      "message": "List the files under /tmp and tell me how many there are.",
+      "max_turns": 8,
+      "timeout_s": 150,
+      "quiet_s": 12,
+      "assertions": [
+        { "type": "result_matches", "pattern": "\\d" },
+        { "type": "calls_exclude", "pattern": "harness::spawn|engine::register_trigger" },
+        { "type": "calls_exclude", "pattern": "directory::skills::get" }
+      ]
+    },
+    {
+      "id": "discovery-workers",
+      "message": "Which workers are connected to the engine right now? Just give me the names.",
+      "max_turns": 8,
+      "timeout_s": 150,
+      "quiet_s": 12,
+      "assertions": [
+        { "type": "calls_include", "pattern": "engine::workers::list" },
+        { "type": "calls_exclude", "pattern": "harness::spawn" }
+      ]
+    },
+    {
+      "id": "fanout-state-report",
+      "message": "Spawn 3 sub-agents in parallel. Each writes one interesting fact about the number 7, 8, and 9 respectively to state. Report back to me with all three facts once they are all in.",
+      "max_turns": 16,
+      "timeout_s": 480,
+      "quiet_s": 90,
+      "assertions": [
+        { "type": "calls_include", "pattern": "harness::spawn" },
+        { "type": "result_matches", "pattern": "7[\\s\\S]*8[\\s\\S]*9" },
+        {
+          "type": "call_order",
+          "before": "directory::skills::get",
+          "after": "harness::spawn",
+          "arms": ["candidate"],
+          "optional": true,
+          "note": "playbook pull should precede orchestration when it happens; engine-native in-turn spawns may legitimately skip it"
+        }
+      ]
+    },
+    {
+      "id": "bulk-two-fields",
+      "message": "Fetch https://iii.dev/docs/llms-full.txt and tell me two things: its total size in characters, and the first heading line in it.",
+      "max_turns": 8,
+      "timeout_s": 300,
+      "quiet_s": 12,
+      "assertions": [
+        { "type": "calls_include", "pattern": "fp::pipe" },
+        { "type": "calls_exclude", "pattern": "^web::fetch$", "note": "a direct fetch floods ~270k tokens of body into the context" },
+        { "type": "result_matches", "pattern": "1[,.]?000[,.]?948|Helpers" }
+      ]
+    },
+    {
+      "id": "registry-lookup-no-install",
+      "message": "Do we have any worker available that can send email? Do not install anything, just tell me what exists.",
+      "max_turns": 8,
+      "timeout_s": 150,
+      "quiet_s": 12,
+      "assertions": [
+        { "type": "calls_exclude", "pattern": "worker::add" }
+      ]
+    }
+  ]
+}