From a6f2a8c976bf506e35baabea7d1998ebe629e40e Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Thu, 13 Aug 2026 17:13:08 +0200 Subject: [PATCH 01/12] feat(framework): pnpm local run, with provenance receipts and pre-spend gates Runs an eval against YOUR inputs and records which world it measured. Nothing here mutates a git tree, so it is safe to run beside in-flight work in another worktree. pnpm local run [--experiment ] [--runs N] [--mcp ] pnpm local experiments Every run writes results-local/.treatment.json: host sha and dirty-file count, plus the --mcp override's own git state when one is passed. `--mcp` is sugar over the SUPABASE_MCP_SERVER_PATH resolution added in the previous commit, and it accepts either the mcp monorepo root or the server package dir. The gates are the point. The harness SKIPs an experiment with exit 0 when credentials are missing, so without them a misconfigured world surfaces only after a paid agent run: - eval metadata parses via parseEvalMarkdown, and the eval dir exists - the experiment exists, listing what is available when it does not - ANTHROPIC_API_KEY is set AND non-empty. Set-but-empty is its own case because `node --env-file` never overrides an existing var, so a stray `export ANTHROPIC_API_KEY=` silently shadows .env (observed live). - OPENAI_API_KEY is present for judge-scored evals. The judge is an OpenAI grader regardless of the agent under test, and a missing key would otherwise surface after the agent run finished. - the experiment's declared skills exist in this checkout, or the treatment would quietly run skill-less - the --mcp path exists and is actually built, warning on fixture drift when the build's version differs from the pin published-log.ts parses the `git log` record for a published export. It is split out so the smoke suite can exercise the merge-commit case directly: `%P` expands to every parent, so a space-split would put a second sha where the timestamp belongs, and origin/main has no merge touching an export for an end-to-end check to catch it. `pnpm test:local` is a zero-cost smoke suite: 11 checks, no model call and no docker. LOCAL_EVAL_CMD fakes the run, LOCAL_NO_FETCH skips the fetch, and LOCAL_RESULTS_ROOT redirects every write into a temp sandbox so a real in-flight run cannot be clobbered. Verified: smoke 11/11, tsc and biome clean. --- .gitignore | 2 + apps/framework/package.json | 4 +- apps/framework/scripts/local.ts | 458 ++++++++++++++++++++++++ apps/framework/scripts/published-log.ts | 52 +++ apps/framework/scripts/smoke-local.ts | 284 +++++++++++++++ package.json | 3 +- packages/core/src/eval-metadata.ts | 1 + 7 files changed, 802 insertions(+), 2 deletions(-) create mode 100755 apps/framework/scripts/local.ts create mode 100644 apps/framework/scripts/published-log.ts create mode 100644 apps/framework/scripts/smoke-local.ts diff --git a/.gitignore b/.gitignore index 5dc75e68..66c7a855 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ evals/*/local/supabase/.branches/ results/*/ .sync-tmp/ +# local-dev runner (apps/framework/scripts/local.ts) +/results-local/ diff --git a/apps/framework/package.json b/apps/framework/package.json index 5b282780..2be1aa36 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -14,7 +14,9 @@ "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", - "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" + "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts", + "local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/local.ts", + "test:local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-local.ts" }, "dependencies": { "@ai-sdk/anthropic": "catalog:", diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts new file mode 100755 index 00000000..472a5a7b --- /dev/null +++ b/apps/framework/scripts/local.ts @@ -0,0 +1,458 @@ +#!/usr/bin/env tsx +/** + * local.ts — local-dev runner. Run evals against YOUR inputs (an edited skills + * tree, a local MCP build) with provenance receipts. + * + * pnpm local run [--experiment ] [--runs N] [--mcp ] + * pnpm local experiments + * + * Design notes: + * - Treatment-only: nothing here ever mutates a git tree, so concurrent + * sessions/worktrees cannot interfere and in-flight work is never at risk. + * - Explicit over magic: this does not build your MCP checkout for you; it + * reports what world it measured. Build it with `pnpm build` in your mcp + * checkout and pass `--mcp`. + * - Gates run before any model call, because the harness SKIPs an experiment + * with exit 0 on missing credentials and a wasted agent run costs real money. + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs, type ParseArgsConfig } from 'node:util'; +import { + getExperimentDisplayMetadata, + MCP_SERVER_VERSION, + type ExperimentConfig, +} from '@supabase-evals/core'; +import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; +import { + rawEvalResultSchema, + type RawEvalResult, +} from '@supabase-evals/core/eval-metadata'; +import { parsePublishedLog, PUBLISHED_LOG_FORMAT } from './published-log.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '..', '..', '..'); +// test seam: the smoke suite redirects ALL outputs into a temp sandbox so it +// can never clobber a real (possibly in-flight) run's results/receipts +const RESULTS_ROOT = process.env.LOCAL_RESULTS_ROOT ?? ROOT; +const OUT_DIR = join(RESULTS_ROOT, 'results-local'); +// suite name -> published export file; the values double as the full load list +const PUBLISHED_EXPORTS: Record = { + regression: 'apps/web/src/data/regression-eval-results.json', + benchmark: 'apps/web/src/data/eval-results.json', +}; +const DEFAULT_EXPERIMENT = 'claude-code-sonnet-5'; + +function fail(msg: string): never { + console.error(msg); + process.exit(1); +} + +// ---------- git helpers (plain child_process; cross-platform) ---------- + +function git(args: string[], cwd: string = ROOT): string { + return execFileSync('git', args, { cwd, maxBuffer: 1 << 28 }) + .toString() + .trim(); +} + +function tryGit(args: string[], cwd: string = ROOT): string | undefined { + try { + return git(args, cwd); + } catch { + return undefined; + } +} + +// ---------- provenance receipts ---------- + +type Provenance = { + generatedAt: string; + host: { sha?: string; branch?: string; dirtyFiles: number }; + mcpOverride?: { path: string; sha?: string; dirtyFiles?: number }; + platform: string; +}; + +function collectProvenance(mcpPath?: string): Provenance { + const dirty = (cwd: string) => + (tryGit(['status', '--porcelain'], cwd) ?? '').split('\n').filter(Boolean) + .length; + const p: Provenance = { + generatedAt: new Date().toISOString(), + host: { + sha: tryGit(['rev-parse', 'HEAD']), + branch: tryGit(['rev-parse', '--abbrev-ref', 'HEAD']), + dirtyFiles: dirty(ROOT), + }, + platform: `${process.platform}/${process.arch} node ${process.version}`, + }; + if (mcpPath) { + const inRepo = tryGit(['rev-parse', '--show-toplevel'], mcpPath); + p.mcpOverride = { + path: mcpPath, + sha: inRepo ? tryGit(['rev-parse', 'HEAD'], mcpPath) : undefined, + dirtyFiles: inRepo ? dirty(inRepo) : undefined, + }; + } + return p; +} + +// ---------- published baselines (compare mode) ---------- + +type PublishedFile = { + file: string; + rows: RawEvalResult[]; + commit: string; + parent: string; + committedAt: string; +}; + +/** Load one published export file from origin/main with its commit metadata. */ +function loadPublishedFile(file: string): PublishedFile | undefined { + let rows: RawEvalResult[]; + try { + rows = JSON.parse(git(['show', `origin/main:${file}`])); + } catch { + return undefined; + } + const line = git([ + 'log', + 'origin/main', + '-1', + PUBLISHED_LOG_FORMAT, + '--', + file, + ]); + return { file, rows, ...parsePublishedLog(line) }; +} + +/** Fetch origin/main so the published exports are current; warn-and-continue offline. */ +function fetchMain() { + if (process.env.LOCAL_NO_FETCH) return; + try { + git(['fetch', '-q', 'origin', 'main']); + } catch { + console.error( + 'warning: could not fetch origin/main — comparing against the local ref, which may be stale' + ); + } +} + +/** Load every published export once; callers share the result. */ +function loadPublished(): PublishedFile[] { + return Object.values(PUBLISHED_EXPORTS).flatMap( + (f) => loadPublishedFile(f) ?? [] + ); +} + +// ---------- eval validation (fail before spending) ---------- + +function validateEvals(evalIds: string[]) { + for (const id of evalIds) { + const promptPath = join(ROOT, 'evals', id, 'PROMPT.md'); + if (!existsSync(promptPath)) + fail(`no eval at evals/${id} (PROMPT.md missing)`); + try { + parseEvalMarkdown( + readFileSync(promptPath, 'utf8'), + `evals/${id}/PROMPT.md` + ); + } catch (err) { + fail( + `eval metadata invalid — fix evals/${id}/PROMPT.md before spending on runs\n${err instanceof Error ? err.message : String(err)}` + ); + } + } +} + +function validateExperiment(experiment: string) { + if (!existsSync(join(ROOT, 'experiments', `${experiment}.ts`))) { + const available = readdirSync(join(ROOT, 'experiments')) + .filter((f) => f.endsWith('.ts')) + .map((f) => f.replace(/\.ts$/, '')); + fail( + `unknown experiment: ${experiment}\navailable: ${available.join(', ')}\n(or add experiments/${experiment}.ts — see any existing file for the shape)` + ); + } +} + +/** + * The agent itself needs its provider key. Checked here (not just by the + * harness) because the harness SKIPs the experiment with exit 0 on missing + * credentials — the runner would only notice at the no-result check. A + * set-but-EMPTY var counts as missing (node --env-file does not override + * an existing env var, even an empty one, so a stray `export KEY=` in the + * shell silently shadows .env — observed live). + */ +function validateAgentKey() { + if (process.env.ANTHROPIC_API_KEY) return; + fail( + process.env.ANTHROPIC_API_KEY === undefined + ? 'ANTHROPIC_API_KEY not set — add it to .env at the repo root' + : 'ANTHROPIC_API_KEY is set but EMPTY in your shell, which shadows .env (node --env-file never overrides an existing var) — unset it or export a real value' + ); +} + +/** + * Evals whose scorer uses the LLM judge grade the agent's output with an + * OpenAI model — even when the agent under test is Claude. A missing grader + * key otherwise surfaces only AFTER the (paid) agent run, wasting it. + * Textual scan of EVAL.ts; a false positive just asks for a key early. + */ +function validateJudgeKeys(evalIds: string[]) { + if (process.env.OPENAI_API_KEY) return; + const judged = evalIds.filter((id) => { + const scorer = join(ROOT, 'evals', id, 'EVAL.ts'); + return existsSync(scorer) && /\bjudge\b/.test(readFileSync(scorer, 'utf8')); + }); + if (judged.length) + fail( + `these evals score with the LLM judge (OpenAI-backed, regardless of the agent under test): ${judged.join(', ')}\nadd OPENAI_API_KEY to .env at the repo root before running them` + ); +} + +/** + * Accept either the mcp monorepo root or the server package dir for --mcp, + * and refuse pre-spend when the server isn't built (the harness would only + * discover that after eval setup). + */ +function resolveMcpServerPath(raw: string): string { + let p = isAbsolute(raw) ? raw : resolve(process.cwd(), raw); + if (!existsSync(p)) fail(`--mcp path does not exist: ${p}`); + const packageDir = join(p, 'packages', 'mcp-server-supabase'); + if (existsSync(packageDir)) p = packageDir; + if (!existsSync(join(p, 'dist', 'transports', 'stdio.js'))) + fail( + `no built server at ${p} (dist/transports/stdio.js missing) — build it first:\n pnpm install && pnpm build # in the mcp checkout (use \`mise exec --\` if corepack's pnpm mismatches)` + ); + // Fixture-drift heads-up: platform-lite tracks the pinned package version, + // and a local build from a newer line may call endpoints the fixture does + // not serve yet (observed: get_logs moved logs.all -> logs in 0.9.0 while + // the pin and fixture sat at 0.8.x). Warn, don't block. + try { + const local = JSON.parse( + readFileSync(join(p, 'package.json'), 'utf8') + ).version; + if (local && local !== MCP_SERVER_VERSION) + console.error( + `note: local mcp build is v${local}; the harness fixture (platform-lite) tracks the v${MCP_SERVER_VERSION} pin — endpoint drift is possible; judge by tool-call activation, not pass/fail alone` + ); + } catch { + /* unversioned checkout: nothing to compare */ + } + return p; +} + +/** + * The experiment's declared skills must exist in this checkout, or the + * treatment silently runs skill-less against a skills-enabled published + * baseline — a world mismatch, not a comparison. + */ +async function validateSkills(experiment: string) { + // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments) + const mod = await import( + pathToFileURL(join(ROOT, 'experiments', `${experiment}.ts`)).href + ); + const skills: string[] = (mod.default as ExperimentConfig).skills ?? []; + const missing = skills.filter((s) => !existsSync(join(ROOT, 'skills', s))); + if (missing.length) + fail( + `experiment ${experiment} declares skills this checkout is missing: ${missing.join(', ')}\ninitialise the skills submodule first: git submodule update --init` + ); +} + +// ---------- treatment run ---------- + +function runEval( + evalId: string, + experiment: string, + runs: number, + env: Record +): string { + const res = spawnSync( + process.execPath, + [ + '--import', + 'tsx/esm', + join(__dirname, '..', 'harness', 'run-eval.ts'), + '--eval', + evalId, + '--experiment', + experiment, + '--runs', + String(runs), + ], + { + stdio: 'inherit', + cwd: join(__dirname, '..'), + env: { ...process.env, ...env }, + } + ); + if (res.status !== 0) fail(`eval run failed: ${evalId} (exit ${res.status})`); + const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`); + if (!existsSync(resultPath)) + fail( + `no result at results/${experiment}/${evalId}.json — check the eval/experiment ids` + ); + return resultPath; +} + +// test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend) +function fakeRun(evalId: string, experiment: string): string { + const resultPath = join( + RESULTS_ROOT, + 'results', + experiment, + `${evalId}.json` + ); + mkdirSync(dirname(resultPath), { recursive: true }); + const res = spawnSync(process.env.LOCAL_EVAL_CMD as string, { + shell: true, + stdio: 'inherit', + env: { ...process.env, RES: resultPath, EVAL: evalId }, + }); + if (res.status !== 0) fail(`LOCAL_EVAL_CMD failed for ${evalId}`); + return resultPath; +} + +// ---------- reporting ---------- + +function reportRow( + label: string, + r: RawEvalResult | undefined, + extra: string +): string { + const checks = r?.checks ?? []; + const checksSummary = `${checks.filter((x) => x.passed).length}/${checks.length}`; + const docsCalls = r?.docs?.calls?.length ?? 0; + return `${label.padEnd(10)} passed=${String(r?.passed).padEnd(5)} checks=${checksSummary.padEnd(6)} docs.calls=${String(docsCalls).padEnd(3)} ${extra}`; +} + +/** Run one eval in the treatment world, write its receipt, report. */ +function runTreatment( + id: string, + experiment: string, + runs: number, + opts: { env: Record; mcpPath?: string } +): void { + const { env, mcpPath } = opts; + console.log( + `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}) ==` + ); + const resultPath = process.env.LOCAL_EVAL_CMD + ? fakeRun(id, experiment) + : runEval(id, experiment, runs, env); + + const parsed = rawEvalResultSchema.safeParse( + JSON.parse(readFileSync(resultPath, 'utf8')) + ); + if (!parsed.success) + fail( + `result at ${resultPath} does not match the eval result contract:\n${parsed.error.message}` + ); + const result = parsed.data; + const receipt = { + ...result, + provenance: collectProvenance(mcpPath), + }; + writeFileSync( + join(OUT_DIR, `${id}.treatment.json`), + `${JSON.stringify(receipt, null, 1)}\n` + ); + + console.log(`\n=== local run: ${id} (${experiment}) ===`); + console.log(reportRow('treatment', result, 'your world')); + console.log(`saved: results-local/${id}.treatment.json`); +} + +// ---------- subcommands ---------- + +async function cmdExperiments() { + const published = new Set( + loadPublished().flatMap((f) => f.rows.map((r) => r.experiment)) + ); + console.log( + `${'EXPERIMENT'.padEnd(36)} ${'AGENT'.padEnd(12)} ${'MODEL'.padEnd(22)} ${'EFFORT'.padEnd(8)} PUBLISHED` + ); + for (const f of readdirSync(join(ROOT, 'experiments')) + .filter((f) => f.endsWith('.ts')) + .sort()) { + const name = f.replace(/\.ts$/, ''); + // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments) + const mod = await import(pathToFileURL(join(ROOT, 'experiments', f)).href); + const display = getExperimentDisplayMetadata( + mod.default as ExperimentConfig + ); + console.log( + `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes' : '-'}` + ); + } +} + +const RUN_USAGE = + 'usage: pnpm local run [--experiment ] [--runs N] [--mcp ]'; + +async function cmdRun(argv: string[]) { + const parsed = (() => { + try { + return parseArgs({ + args: argv, + options: { + experiment: { type: 'string' }, + runs: { type: 'string' }, + mcp: { type: 'string' }, + }, + allowPositionals: true, + }); + } catch (err) { + fail(`${err instanceof Error ? err.message : String(err)}\n${RUN_USAGE}`); + } + })(); + const { values, positionals } = parsed; + const experiment = values.experiment ?? DEFAULT_EXPERIMENT; + validateExperiment(experiment); + const evalIds = positionals; + if (!evalIds.length) fail(RUN_USAGE); + + validateEvals(evalIds); + // these gates are spend-relevant only for real runs; the test hook fakes them + if (!process.env.LOCAL_EVAL_CMD) { + validateAgentKey(); + await validateSkills(experiment); + validateJudgeKeys(evalIds); + } + + const env: Record = {}; + const mcpPath = values.mcp ? resolveMcpServerPath(values.mcp) : undefined; + if (mcpPath) env.SUPABASE_MCP_SERVER_PATH = mcpPath; + + mkdirSync(OUT_DIR, { recursive: true }); + for (const id of evalIds) { + const runs = Number(values.runs ?? 1); + runTreatment(id, experiment, runs, { env, mcpPath }); + } +} + +// ---------- entry ---------- + +const [command, ...rest] = process.argv.slice(2); +switch (command) { + case 'run': + await cmdRun(rest); + break; + case 'experiments': + await cmdExperiments(); + break; + default: + fail(`usage: pnpm local ... + run run eval(s) in your world (skills tree as-is; --mcp override) + experiments list experiments (agent, model, effort, published availability)`); +} diff --git a/apps/framework/scripts/published-log.ts b/apps/framework/scripts/published-log.ts new file mode 100644 index 00000000..8c895c2a --- /dev/null +++ b/apps/framework/scripts/published-log.ts @@ -0,0 +1,52 @@ +/** + * Parsing for the one-line `git log` record describing a published export's + * newest commit. Split out of local.ts so the smoke suite can exercise the + * merge-commit case directly: local.ts reads `origin/main`, whose history has + * no qualifying merge today, so an end-to-end check cannot reach it. + */ + +/** The `--format` string `parsePublishedLog` expects. Tab-separated on purpose. */ +export const PUBLISHED_LOG_FORMAT = '--format=%H%x09%P%x09%cI'; + +export type PublishedLog = { + /** Full sha of the newest commit touching the export. */ + commit: string; + /** First parent — the mainline, i.e. the ref the scheduled run built on. */ + parent: string; + /** Committer date, ISO 8601. */ + committedAt: string; +}; + +/** + * Parse one `PUBLISHED_LOG_FORMAT` line. + * + * Tab-separated because `%P` expands to EVERY parent, space-separated. Splitting + * the whole line on spaces therefore binds the second parent to `committedAt` on + * a merge commit, and a sha where an ISO date belongs poisons the baseline sort + * (`Date.parse` -> NaN) and prints "NaNd old" into the receipt. Path-limited log + * simplification hides most merges, but not one that changed the export relative + * to both parents — so this is reachable, not theoretical. + */ +export function parsePublishedLog(line: string): PublishedLog { + const bad = (why: string): never => { + throw new Error( + `unparseable published log line (${why}): ${JSON.stringify(line)}` + ); + }; + // Strip only the trailing newline. `trim()` would also eat a trailing empty + // field's separator, turning a malformed record into a confusing field count + // instead of a precise "committedAt is not a date". + const fields = line.replace(/\r?\n$/, '').split('\t'); + if (fields.length !== 3) + bad(`expected 3 tab-separated fields, got ${fields.length}`); + const [commit, parents, committedAt] = fields; + const isSha = (s: string) => /^[0-9a-f]{40}$/.test(s); + if (!isSha(commit)) bad('commit is not a sha'); + if (Number.isNaN(Date.parse(committedAt))) bad('committedAt is not a date'); + // A parentless commit means the premise behind `parent` (the ref the + // scheduled run built on) does not hold, so refuse rather than record ''. + const list = parents === '' ? [] : parents.split(' '); + if (!list.length) bad('commit has no parent'); + if (!list.every(isSha)) bad('parent is not a sha'); + return { commit, parent: list[0], committedAt }; +} diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts new file mode 100644 index 00000000..389cb118 --- /dev/null +++ b/apps/framework/scripts/smoke-local.ts @@ -0,0 +1,284 @@ +/** + * Zero-cost smoke test for the local-dev runner (scripts/local.ts). + * + * Fakes the eval run via LOCAL_EVAL_CMD (no model spend, no docker) and + * reads REAL published baselines from origin/main (no fetch: LOCAL_NO_FETCH). + * + * pnpm --filter @supabase-evals/framework test:local + */ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { parsePublishedLog, PUBLISHED_LOG_FORMAT } from './published-log.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(__dirname, '..', '..', '..'); +// every output lands in a disposable sandbox — never the checkout's real +// results/ or results-local/ (an in-flight manual run may own those) +const SANDBOX = mkdtempSync(join(tmpdir(), 'smoke-local-')); +const OUT = join(SANDBOX, 'results-local'); +const EXPERIMENT = 'claude-code-sonnet-5'; + +// a published, currently-existing eval id — resolved dynamically so the test +// doesn't rot when the published set changes +const published = JSON.parse( + execFileSync( + 'git', + ['show', 'origin/main:apps/web/src/data/regression-eval-results.json'], + { cwd: ROOT, maxBuffer: 1 << 28 } + ).toString() +) as Array<{ experiment: string; eval: string }>; +const EVAL = published.find( + (r) => r.experiment === EXPERIMENT && existsSync(join(ROOT, 'evals', r.eval)) +)?.eval; +assert.ok(EVAL, 'no published eval with a local evals/ dir found'); + +// LOCAL_EVAL_CMD contract: write a result JSON to $RES for eval $EVAL. +// A script file sidesteps per-platform shell quoting entirely. +const fakeScript = join(SANDBOX, 'fake-eval.cjs'); +writeFileSync( + fakeScript, + `const fs = require('node:fs'); +const path = require('node:path'); +fs.mkdirSync(path.dirname(process.env.RES), { recursive: true }); +fs.writeFileSync( + process.env.RES, + JSON.stringify({ + eval: process.env.EVAL, + experiment: '${EXPERIMENT}', + passed: true, + checks: [{ name: 'x', passed: true }], + }) +); +` +); +const FAKE = `${JSON.stringify(process.execPath)} ${JSON.stringify(fakeScript)}`; + +function local(args: string[], env: Record = {}) { + const res = spawnSync( + process.execPath, + ['--import', 'tsx/esm', join(__dirname, 'local.ts'), ...args], + { + cwd: join(__dirname, '..'), + encoding: 'utf8', + timeout: 60_000, // a regressed pre-spend gate must never reach a real agent run + env: { + ...process.env, + LOCAL_NO_FETCH: '1', + LOCAL_RESULTS_ROOT: SANDBOX, + LOCAL_EVAL_CMD: FAKE, + FORCE_COLOR: '0', + ...env, + }, + } + ); + return { out: `${res.stdout}\n${res.stderr}`, status: res.status }; +} + +let passed = 0; +function ck(name: string, fn: () => void) { + try { + fn(); + passed++; + } catch (err) { + console.error(`FAIL: ${name}`); + throw err; + } +} + +// --- published-log parsing: the merge-commit case origin/main cannot reach --- +// Built here rather than asserted against real history: main has no merge that +// touches a published export, so an end-to-end check would pass either way. +{ + const repo = join(SANDBOX, 'merge-parse-repo'); + const file = 'exports.json'; + const g = (args: string[]) => + execFileSync('git', args, { cwd: repo, encoding: 'utf8' }).trim(); + mkdirSync(repo, { recursive: true }); + g(['init', '-q', '-b', 'main']); + g(['config', 'user.email', 'smoke@local']); + g(['config', 'user.name', 'smoke']); + const commitFile = (body: string, msg: string) => { + writeFileSync(join(repo, file), body); + g(['add', file]); + g(['commit', '-qm', msg]); + }; + commitFile('[{"eval":"base"}]\n', 'base'); + g(['checkout', '-q', '-b', 'side']); + commitFile('[{"eval":"side"}]\n', 'side'); + g(['checkout', '-q', 'main']); + commitFile('[{"eval":"main"}]\n', 'main edit'); + // Expected to conflict: we resolve to content differing from BOTH parents so + // path-limited history simplification keeps this merge in `git log -- `. + // The conflict is the point, so assert a merge is genuinely in progress rather + // than discarding the exit status and hoping. + const merge = spawnSync('git', ['merge', 'side'], { + cwd: repo, + encoding: 'utf8', + }); + assert.notEqual( + merge.status, + 0, + `expected \`git merge side\` to conflict, got clean exit: ${merge.stdout}${merge.stderr}` + ); + assert.ok( + existsSync(join(repo, '.git', 'MERGE_HEAD')), + 'no MERGE_HEAD: the next commit would not be a merge commit' + ); + commitFile('[{"eval":"merged"}]\n', 'merge side into main'); + + const line = g(['log', 'main', '-1', PUBLISHED_LOG_FORMAT, '--', file]); + + ck('published log line for a merge commit really has 2 parents', () => { + const [, parents] = line.split('\t'); + assert.equal(parents.split(' ').length, 2, `expected a merge: ${line}`); + }); + + ck('merge commit parses to the mainline parent and a real date', () => { + const parsed = parsePublishedLog(line); + assert.match(parsed.commit, /^[0-9a-f]{40}$/); + assert.match(parsed.parent, /^[0-9a-f]{40}$/); + assert.equal(parsed.parent, g(['rev-parse', 'main^1'])); + assert.ok( + !Number.isNaN(Date.parse(parsed.committedAt)), + `committedAt is not a date: ${parsed.committedAt}` + ); + }); + + ck('the old space-split is what this guards against', () => { + // Reproduce the pre-fix parse to prove the regression is real: `%P` puts a + // second sha where the timestamp belongs, and Date.parse yields NaN. + const spaceSplit = g([ + 'log', + 'main', + '-1', + '--format=%H %P %cI', + '--', + file, + ]).split(' '); + assert.equal(spaceSplit.length, 4); + assert.ok(Number.isNaN(Date.parse(spaceSplit[2]))); + }); + + ck('malformed published log lines are refused, not guessed', () => { + assert.throws(() => parsePublishedLog('only-one-field'), /expected 3/); + assert.throws( + () => parsePublishedLog('nothex\tdead\t2026-01-01'), + /not a sha/ + ); + assert.throws( + () => parsePublishedLog(`${'a'.repeat(40)}\t${'b'.repeat(40)}\tnope`), + /not a date/ + ); + assert.throws( + () => parsePublishedLog(`${'a'.repeat(40)}\t\t2026-01-01T00:00:00Z`), + /no parent/ + ); + }); +} + +// --- refusals happen pre-spend, with actionable messages --- +{ + const r = local(['run', EVAL, '--experiment', 'bogus-model']); + ck('unknown experiment refused with the available list', () => { + assert.equal(r.status, 1); + assert.match(r.out, /unknown experiment: bogus-model/); + assert.match(r.out, /claude-code-sonnet-5/); + }); +} +{ + const r = local(['run', 'not-an-eval-dir']); + ck('missing eval dir refused', () => { + assert.equal(r.status, 1); + assert.match(r.out, /no eval at evals\/not-an-eval-dir/); + }); +} + +// --- run: no baseline required (custom evals), receipt only --- +{ + const r = local(['run', EVAL]); + ck('run works without published baseline machinery', () => { + assert.equal(r.status, 0); + assert.match(r.out, new RegExp(`=== local run: ${EVAL}`)); + assert.doesNotMatch(r.out, /published /); + assert.match(r.out, /saved: results-local\//); + }); +} + +// --- mcp override path validation --- +{ + const r = local(['run', EVAL, '--mcp', '/definitely/not/a/path']); + ck('bad --mcp path refused pre-spend', () => { + assert.equal(r.status, 1); + assert.match(r.out, /--mcp path does not exist/); + }); +} + +// --- mcp override: monorepo root resolves to the server package; unbuilt refused --- +{ + const fake = join(SANDBOX, '.smoke-mcp-checkout'); + const pkg = join(fake, 'packages', 'mcp-server-supabase'); + mkdirSync(join(pkg, 'dist', 'transports'), { recursive: true }); + + const unbuilt = local(['run', EVAL, '--mcp', fake]); + ck('unbuilt mcp checkout refused pre-spend with build hint', () => { + assert.equal(unbuilt.status, 1); + assert.match(unbuilt.out, /no built server at .*mcp-server-supabase/); + assert.match(unbuilt.out, /pnpm install && pnpm build/); + }); + + writeFileSync( + join(pkg, 'dist', 'transports', 'stdio.js'), + '// smoke fixture\n' + ); + const built = local(['run', EVAL, '--mcp', fake]); + ck('monorepo root resolves to the server package dir', () => { + assert.equal(built.status, 0); + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8') + ); + assert.match( + receipt.provenance.mcpOverride.path, + /packages[/\\]mcp-server-supabase$/ + ); + }); + rmSync(fake, { recursive: true, force: true }); +} + +// --- judge-key gate: refused pre-spend, before any agent spawn --- +{ + // needs an eval whose scorer really uses the judge; EVAL may not + const judgedEval = published.find( + (row) => + row.experiment === EXPERIMENT && + existsSync(join(ROOT, 'evals', row.eval, 'EVAL.ts')) && + /\bjudge\b/.test( + readFileSync(join(ROOT, 'evals', row.eval, 'EVAL.ts'), 'utf8') + ) + )?.eval; + assert.ok(judgedEval, 'no judged eval found in the published set'); + const r = local(['run', judgedEval], { + LOCAL_EVAL_CMD: '', + OPENAI_API_KEY: '', + }); + ck('judged eval without OPENAI_API_KEY refused pre-spend', () => { + assert.equal(r.status, 1); + assert.match(r.out, /score with the LLM judge/); + assert.match(r.out, /add OPENAI_API_KEY/); + }); +} + +// cleanup: everything lived in the sandbox +rmSync(SANDBOX, { recursive: true, force: true }); + +console.log(`smoke-local: ${passed} checks passed`); diff --git a/package.json b/package.json index ab7f505e..dbe8591f 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp", "demo:executor": "pnpm --filter @supabase-evals/framework demo:executor", "format": "biome check --write . && pnpm --filter @supabase-evals/web format", - "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check" + "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check", + "local": "pnpm --filter @supabase-evals/framework local" }, "dependencies": { "@ai-sdk/anthropic": "catalog:", diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 66c3f8bf..6d94b683 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -347,6 +347,7 @@ const evalResultShape = { // Raw result files may carry extra fields we don't model; tolerate them. export const rawEvalResultSchema = z.looseObject(evalResultShape); +export type RawEvalResult = z.infer; // Web-facing result; a clean strict object so its inferred type stays usable. export const evalResultSchema = z.object({ From c100322620762ce6f30844c056db6a0f868f76c5 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 11:10:59 +0200 Subject: [PATCH 02/12] feat(eval): add strict local-run gates --- apps/framework/harness/run-eval.ts | 249 +++++++++++++++++++++-------- 1 file changed, 186 insertions(+), 63 deletions(-) diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0c43b29f..ae81d246 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -1,4 +1,5 @@ #!/usr/bin/env tsx +import { execFileSync, spawnSync } from 'node:child_process'; import { cpSync, existsSync, @@ -10,7 +11,7 @@ import { statSync, writeFileSync, } from 'node:fs'; -import { join, dirname, relative } from 'node:path'; +import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { jsonSchema, tool, type ToolSet } from 'ai'; import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; @@ -32,8 +33,9 @@ import { viteBuild, vitestRun } from './project-runner.js'; import { buildDocsResult, buildSkillResult, - rehydrateTruncatedDocsResults, getExperimentDisplayMetadata, + MCP_SERVER_VERSION, + rehydrateTruncatedDocsResults, supabaseMcpServerMounts, } from '@supabase-evals/core'; import type { @@ -53,6 +55,8 @@ import type { const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = join(__dirname, '..', '..', '..'); +const RESULTS_ROOT = process.env.LOCAL_RESULTS_ROOT ?? ROOT; +const SKILLS_ROOT = process.env.LOCAL_SKILLS_ROOT ?? join(ROOT, 'skills'); // Fixed identifiers for the mocked hosted project a local-stack eval links to. // Both must satisfy the CLI's format checks: ref is `^[a-z]{20}$`, token is @@ -65,6 +69,8 @@ const args = new Set(rawArgs); const FORCE = !args.has('--skip-existing'); const SMOKE = args.has('--smoke'); const DRY = args.has('--dry'); +const STRICT = args.has('--strict'); +const MCP_PATH = readFlag(rawArgs, 'mcp'); const EXPERIMENT_FILTERS = readRepeatedFlag(rawArgs, 'experiment').map( normalizeExperimentName ); @@ -163,7 +169,7 @@ type ToolsSkill = { name: string; description: string; body: string }; function loadToolsSkills(skillNames: string[]): ToolsSkill[] { const skills: ToolsSkill[] = []; for (const name of skillNames) { - const p = join(ROOT, 'skills', name, 'SKILL.md'); + const p = join(SKILLS_ROOT, name, 'SKILL.md'); if (!existsSync(p)) { console.warn( `SKILL ${name} not found at skills/${name} — ensure the submodule is initialised (\`git submodule update --init\`); skipping` @@ -250,7 +256,7 @@ function resolveSkillSources( ): Array<{ name: string; dir: string }> { const sources: Array<{ name: string; dir: string }> = []; for (const name of skillNames) { - const dir = join(ROOT, 'skills', name); + const dir = join(SKILLS_ROOT, name); if (!existsSync(dir)) { console.warn( `SKILL ${name} not found at skills/${name} — ensure the submodule is initialised (\`git submodule update --init\`); skipping` @@ -263,12 +269,12 @@ function resolveSkillSources( } function resultPath(modelName: string, ev: Pick) { - return join(ROOT, 'results', modelName, `${ev.id}.json`); + return join(RESULTS_ROOT, 'results', modelName, `${ev.id}.json`); } function workspacePath(modelName: string, evalId: string, attempt: number) { return join( - ROOT, + RESULTS_ROOT, 'results', modelName, evalId, @@ -339,22 +345,21 @@ function disposable }>( }, }); } +type RunResult = ScoreResult & { + attempts: number; + skills: SkillResult; + docs: DocsResult; + toolCalls: ToolCallRecord[]; + transcript: TranscriptPart[]; + agentReport: string; + stoppedReason: string; +}; async function runOne( expName: string, exp: ExperimentConfig, ev: EvalManifest -): Promise< - ScoreResult & { - attempts: number; - skills: SkillResult; - docs: DocsResult; - toolCalls: ToolCallRecord[]; - transcript: TranscriptPart[]; - agentReport: string; - stoppedReason: string; - } -> { +): Promise { const prompt = parseEvalMarkdown( readFileSync(ev.promptPath, 'utf8'), ev.promptPath @@ -630,39 +635,151 @@ async function runConcurrent( ); await Promise.all(workers); } +type Provenance = { + generatedAt: string; + host: { sha?: string; branch?: string; dirtyFiles: number }; + mcpOverride?: { path: string; sha?: string; dirtyFiles?: number }; + platform: string; +}; + +function tryGit(args: string[], cwd: string): string | undefined { + try { + return execFileSync('git', args, { cwd, maxBuffer: 1 << 28 }) + .toString() + .trim(); + } catch { + return undefined; + } +} + +function collectProvenance(mcpPath?: string): Provenance { + const dirty = (cwd: string) => + (tryGit(['status', '--porcelain'], cwd) ?? '').split('\n').filter(Boolean) + .length; + const provenance: Provenance = { + generatedAt: new Date().toISOString(), + host: { + sha: tryGit(['rev-parse', 'HEAD'], ROOT), + branch: tryGit(['rev-parse', '--abbrev-ref', 'HEAD'], ROOT), + dirtyFiles: dirty(ROOT), + }, + platform: `${process.platform}/${process.arch} node ${process.version}`, + }; + if (mcpPath) { + const repository = tryGit(['rev-parse', '--show-toplevel'], mcpPath); + provenance.mcpOverride = { + path: mcpPath, + sha: repository ? tryGit(['rev-parse', 'HEAD'], repository) : undefined, + dirtyFiles: repository ? dirty(repository) : undefined, + }; + } + return provenance; +} + +function resolveMcpServerPath(raw: string): string { + let path = isAbsolute(raw) ? raw : resolve(process.cwd(), raw); + if (!existsSync(path)) throw new Error(`--mcp path does not exist: ${path}`); + const packageDir = join(path, 'packages', 'mcp-server-supabase'); + if (existsSync(packageDir)) path = packageDir; + if (!existsSync(join(path, 'dist', 'transports', 'stdio.js'))) { + throw new Error( + `no built server at ${path} (dist/transports/stdio.js missing) — build it first:\n pnpm install && pnpm build` + ); + } + try { + const localVersion = JSON.parse( + readFileSync(join(path, 'package.json'), 'utf8') + ).version; + if (localVersion && localVersion !== MCP_SERVER_VERSION) { + console.error( + `note: local mcp build is v${localVersion}; the harness fixture (platform-lite) tracks the v${MCP_SERVER_VERSION} pin — endpoint drift is possible` + ); + } + } catch { + // An unversioned checkout is valid when it has the expected built entry. + } + return realpathSync(path); +} + +function validateJudgeKeys(evals: readonly EvalManifest[]) { + if (process.env.OPENAI_API_KEY || DRY) return; + const judged = evals + .filter( + (ev) => + existsSync(ev.evalPath) && + /\bjudge\b/.test(readFileSync(ev.evalPath, 'utf8')) + ) + .map((ev) => ev.id); + if (judged.length > 0) { + throw new Error( + `these evals score with the LLM judge (OpenAI-backed, regardless of the agent under test): ${judged.join(', ')}\nadd OPENAI_API_KEY to .env at the repo root before running them` + ); + } +} + +function validateStrictSkills( + experiment: string, + skillNames: readonly string[] +) { + if (!STRICT) return; + const missing = skillNames.filter( + (name) => !existsSync(join(SKILLS_ROOT, name)) + ); + if (missing.length > 0) { + throw new Error( + `experiment ${experiment} declares skills this checkout is missing: ${missing.join(', ')}\ninitialise the skills submodule first: git submodule update --init` + ); + } +} + +function fakeRun(out: string, experiment: string, evalId: string): RunResult { + const command = process.env.LOCAL_EVAL_CMD; + if (!command) throw new Error('LOCAL_EVAL_CMD is required'); + mkdirSync(dirname(out), { recursive: true }); + const result = spawnSync(command, { + shell: true, + stdio: 'inherit', + env: { ...process.env, RES: out, EVAL: evalId, EXPERIMENT: experiment }, + }); + if (result.status !== 0) { + throw new Error(`LOCAL_EVAL_CMD failed for ${experiment} x ${evalId}`); + } + return JSON.parse(readFileSync(out, 'utf8')) as RunResult; +} async function main() { - if (rawArgs.filter((a) => a !== '--')[0] === 'list') { + if (rawArgs.filter((arg) => arg !== '--')[0] === 'list') { const experiments = await loadExperiments(); let filtered = EXPERIMENT_SUITE_FILTERS.length > 0 ? experiments.filter( - (e) => - e.config.suite !== undefined && - e.config.suite.some((suite) => + (experiment) => + experiment.config.suite !== undefined && + experiment.config.suite.some((suite) => EXPERIMENT_SUITE_FILTERS.includes(suite) ) ) : experiments; if (EVAL_FILTERS.length > 0) { - // Drop experiments that would skipEval every requested eval, so callers - // building an experiment x eval matrix (e.g. the eval-refresh workflow) - // don't plan a pair that will produce no results — and no artifact — - // to upload. - const evals = discoverEvals().filter((ev) => - EVAL_FILTERS.includes(ev.id) + const evals = discoverEvals().filter((evaluation) => + EVAL_FILTERS.includes(evaluation.id) ); filtered = filtered.filter(({ config }) => - evals.some((ev) => !config.skipEval?.(ev)) + evals.some((evaluation) => !config.skipEval?.(evaluation)) ); } - console.log(JSON.stringify(filtered.map((e) => e.name))); + console.log(JSON.stringify(filtered.map((experiment) => experiment.name))); return; } + const mcpPath = MCP_PATH ? resolveMcpServerPath(MCP_PATH) : undefined; + if (mcpPath) process.env.SUPABASE_MCP_SERVER_PATH = mcpPath; + const allExperiments = await loadExperiments(); if (EXPERIMENT_FILTERS.length > 0) { - const experimentNames = new Set(allExperiments.map(({ name }) => name)); + const experimentNames = new Set( + allExperiments.map((experiment) => experiment.name) + ); const missing = EXPERIMENT_FILTERS.filter( (name) => !experimentNames.has(name) ); @@ -672,26 +789,27 @@ async function main() { } const experiments = allExperiments.filter(({ name, config }) => { - if (EXPERIMENT_FILTERS.length > 0 && !EXPERIMENT_FILTERS.includes(name)) + if (EXPERIMENT_FILTERS.length > 0 && !EXPERIMENT_FILTERS.includes(name)) { return false; + } if ( EXPERIMENT_SUITE_FILTERS.length > 0 && (config.suite === undefined || !config.suite.some((suite) => EXPERIMENT_SUITE_FILTERS.includes(suite))) - ) + ) { return false; + } return true; }); - if (EXPERIMENT_FILTERS.length > 0) { - if (experiments.length === 0) { - throw new Error( - `no experiments matched experiment=${EXPERIMENT_FILTERS.join(',')}` - ); - } + if (EXPERIMENT_FILTERS.length > 0 && experiments.length === 0) { + throw new Error( + `no experiments matched experiment=${EXPERIMENT_FILTERS.join(',')}` + ); } + const evals = discoverEvals(); if (EVAL_FILTERS.length > 0) { - const evalIds = new Set(evals.map((e) => e.id)); + const evalIds = new Set(evals.map((evaluation) => evaluation.id)); const missing = EVAL_FILTERS.filter((evalId) => !evalIds.has(evalId)); if (missing.length > 0) { throw new Error(`no eval matched: ${missing.join(',')}`); @@ -700,17 +818,19 @@ async function main() { const filtered = SMOKE ? Object.values( - evals.reduce>((acc, e) => { - acc[e.stage] ??= e; + evals.reduce>((acc, evaluation) => { + acc[evaluation.stage] ??= evaluation; return acc; }, {}) ) : EVAL_FILTERS.length > 0 - ? evals.filter((e) => EVAL_FILTERS.includes(e.id)) + ? evals.filter((evaluation) => EVAL_FILTERS.includes(evaluation.id)) : evals; const suiteFiltered = SUITE_FILTERS.length > 0 - ? filtered.filter((e) => SUITE_FILTERS.includes(e.suite)) + ? filtered.filter((evaluation) => + SUITE_FILTERS.includes(evaluation.suite) + ) : filtered; if (suiteFiltered.length === 0) { @@ -723,9 +843,7 @@ async function main() { throw new Error(`no evals matched ${filter}`); } - // Suppress noisy supabase-js logs from expected failures; --debug keeps them visible. const stderr = console.error; - if (!DEBUG) console.error = () => undefined; console.log( `${experiments.length} experiment(s), ${suiteFiltered.length} eval(s), ` + @@ -742,8 +860,10 @@ async function main() { if (!DRY) { try { config.agent.assertReady(); - } catch (e) { - stderr(`SKIP ${name} (${e instanceof Error ? e.message : String(e)})`); + } catch (error) { + const message = `${name} (${error instanceof Error ? error.message : String(error)})`; + if (STRICT) throw new Error(message, { cause: error }); + stderr(`SKIP ${message}`); continue; } } @@ -755,15 +875,19 @@ async function main() { continue; } if (ev.mode === 'local-stack' && !config.localStack) { - console.log( - `SKIP ${name} x ${ev.id} (no local stack runtime — add \`localStack: localStackRuntime()\` from "@supabase-evals/sandbox" to experiments/${name}.ts)` - ); + const message = + `${name} x ${ev.id} (no local stack runtime — add ` + + '`localStack: localStackRuntime()` from "@supabase-evals/sandbox" ' + + `to experiments/${name}.ts)`; + if (STRICT) throw new Error(message); + console.log(`SKIP ${message}`); continue; } if (config.skipEval?.(ev)) { console.log(`SKIP ${name} x ${ev.id} (skipEval)`); continue; } + validateStrictSkills(name, ev.metadata.skills ?? config.skills); if (DRY) { console.log(formatPlanLine(name, config, ev)); continue; @@ -772,6 +896,9 @@ async function main() { } } + validateJudgeKeys([...new Set(allWork.map(({ ev }) => ev))]); + if (!DEBUG) console.error = () => undefined; + let localStackTurn = Promise.resolve(); const errored: Error[] = []; @@ -781,7 +908,9 @@ async function main() { console.log(`⏳ RUN ${name} x ${ev.id}`); const run = async () => { try { - const res = await runOne(name, config, ev); + const res = process.env.LOCAL_EVAL_CMD + ? fakeRun(out, name, ev.id) + : await runOne(name, config, ev); mkdirSync(dirname(out), { recursive: true }); const experimentDisplay = getExperimentDisplayMetadata(config); writeFileSync( @@ -794,6 +923,7 @@ async function main() { eval: ev.id, ...ev.metadata, ...res, + provenance: collectProvenance(mcpPath), }, null, 2 @@ -803,24 +933,17 @@ async function main() { console.log( `${res.passed ? '✅ PASS' : '❌ FAIL'} ${name} x ${ev.id} (${formatRunSummary(res)}, ${elapsed}s)\n → ${relative(ROOT, out)}` ); - } catch (e) { - errored.push(new Error(`${name} x ${ev.id}`, { cause: e })); + } catch (error) { + errored.push(new Error(`${name} x ${ev.id}`, { cause: error })); const elapsed = Math.round((Date.now() - start) / 1000); stderr( - `💥 ERR ${name} x ${ev.id}: ${e instanceof Error ? e.message : String(e)} (${elapsed}s)` + `💥 ERR ${name} x ${ev.id}: ${error instanceof Error ? error.message : String(error)} (${elapsed}s)` ); } }; if (ev.mode !== 'local-stack') return run(); - const prev = localStackTurn; - let release!: () => void; - localStackTurn = new Promise((r) => (release = r)); - await prev; - try { - await run(); - } finally { - release(); - } + localStackTurn = localStackTurn.then(run); + await localStackTurn; }; await runConcurrent(allWork, CONCURRENCY, runWork); From 803d350b4acd46ddaedbece3b5cc821d4ba35286 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 11:11:06 +0200 Subject: [PATCH 03/12] refactor(eval): remove separate local CLI --- .gitignore | 2 - apps/framework/package.json | 7 +- apps/framework/scripts/local.ts | 458 ------------------------ apps/framework/scripts/published-log.ts | 52 --- package.json | 3 +- 5 files changed, 4 insertions(+), 518 deletions(-) delete mode 100755 apps/framework/scripts/local.ts delete mode 100644 apps/framework/scripts/published-log.ts diff --git a/.gitignore b/.gitignore index 66c7a855..5dc75e68 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,3 @@ evals/*/local/supabase/.branches/ results/*/ .sync-tmp/ -# local-dev runner (apps/framework/scripts/local.ts) -/results-local/ diff --git a/apps/framework/package.json b/apps/framework/package.json index 2be1aa36..c03b57c6 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -5,9 +5,9 @@ "type": "module", "scripts": { "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner", - "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", - "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", - "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", + "eval": "node --env-file-if-exists=../../.env --import tsx/esm harness/run-eval.ts", + "eval:dry": "node --env-file-if-exists=../../.env --import tsx/esm harness/run-eval.ts --dry", + "eval:smoke": "node --env-file-if-exists=../../.env --import tsx/esm harness/run-eval.ts --smoke", "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", @@ -15,7 +15,6 @@ "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts", - "local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/local.ts", "test:local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-local.ts" }, "dependencies": { diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts deleted file mode 100755 index 472a5a7b..00000000 --- a/apps/framework/scripts/local.ts +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/env tsx -/** - * local.ts — local-dev runner. Run evals against YOUR inputs (an edited skills - * tree, a local MCP build) with provenance receipts. - * - * pnpm local run [--experiment ] [--runs N] [--mcp ] - * pnpm local experiments - * - * Design notes: - * - Treatment-only: nothing here ever mutates a git tree, so concurrent - * sessions/worktrees cannot interfere and in-flight work is never at risk. - * - Explicit over magic: this does not build your MCP checkout for you; it - * reports what world it measured. Build it with `pnpm build` in your mcp - * checkout and pass `--mcp`. - * - Gates run before any model call, because the harness SKIPs an experiment - * with exit 0 on missing credentials and a wasted agent run costs real money. - */ -import { execFileSync, spawnSync } from 'node:child_process'; -import { - existsSync, - mkdirSync, - readFileSync, - readdirSync, - writeFileSync, -} from 'node:fs'; -import { dirname, isAbsolute, join, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { parseArgs, type ParseArgsConfig } from 'node:util'; -import { - getExperimentDisplayMetadata, - MCP_SERVER_VERSION, - type ExperimentConfig, -} from '@supabase-evals/core'; -import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; -import { - rawEvalResultSchema, - type RawEvalResult, -} from '@supabase-evals/core/eval-metadata'; -import { parsePublishedLog, PUBLISHED_LOG_FORMAT } from './published-log.js'; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = resolve(__dirname, '..', '..', '..'); -// test seam: the smoke suite redirects ALL outputs into a temp sandbox so it -// can never clobber a real (possibly in-flight) run's results/receipts -const RESULTS_ROOT = process.env.LOCAL_RESULTS_ROOT ?? ROOT; -const OUT_DIR = join(RESULTS_ROOT, 'results-local'); -// suite name -> published export file; the values double as the full load list -const PUBLISHED_EXPORTS: Record = { - regression: 'apps/web/src/data/regression-eval-results.json', - benchmark: 'apps/web/src/data/eval-results.json', -}; -const DEFAULT_EXPERIMENT = 'claude-code-sonnet-5'; - -function fail(msg: string): never { - console.error(msg); - process.exit(1); -} - -// ---------- git helpers (plain child_process; cross-platform) ---------- - -function git(args: string[], cwd: string = ROOT): string { - return execFileSync('git', args, { cwd, maxBuffer: 1 << 28 }) - .toString() - .trim(); -} - -function tryGit(args: string[], cwd: string = ROOT): string | undefined { - try { - return git(args, cwd); - } catch { - return undefined; - } -} - -// ---------- provenance receipts ---------- - -type Provenance = { - generatedAt: string; - host: { sha?: string; branch?: string; dirtyFiles: number }; - mcpOverride?: { path: string; sha?: string; dirtyFiles?: number }; - platform: string; -}; - -function collectProvenance(mcpPath?: string): Provenance { - const dirty = (cwd: string) => - (tryGit(['status', '--porcelain'], cwd) ?? '').split('\n').filter(Boolean) - .length; - const p: Provenance = { - generatedAt: new Date().toISOString(), - host: { - sha: tryGit(['rev-parse', 'HEAD']), - branch: tryGit(['rev-parse', '--abbrev-ref', 'HEAD']), - dirtyFiles: dirty(ROOT), - }, - platform: `${process.platform}/${process.arch} node ${process.version}`, - }; - if (mcpPath) { - const inRepo = tryGit(['rev-parse', '--show-toplevel'], mcpPath); - p.mcpOverride = { - path: mcpPath, - sha: inRepo ? tryGit(['rev-parse', 'HEAD'], mcpPath) : undefined, - dirtyFiles: inRepo ? dirty(inRepo) : undefined, - }; - } - return p; -} - -// ---------- published baselines (compare mode) ---------- - -type PublishedFile = { - file: string; - rows: RawEvalResult[]; - commit: string; - parent: string; - committedAt: string; -}; - -/** Load one published export file from origin/main with its commit metadata. */ -function loadPublishedFile(file: string): PublishedFile | undefined { - let rows: RawEvalResult[]; - try { - rows = JSON.parse(git(['show', `origin/main:${file}`])); - } catch { - return undefined; - } - const line = git([ - 'log', - 'origin/main', - '-1', - PUBLISHED_LOG_FORMAT, - '--', - file, - ]); - return { file, rows, ...parsePublishedLog(line) }; -} - -/** Fetch origin/main so the published exports are current; warn-and-continue offline. */ -function fetchMain() { - if (process.env.LOCAL_NO_FETCH) return; - try { - git(['fetch', '-q', 'origin', 'main']); - } catch { - console.error( - 'warning: could not fetch origin/main — comparing against the local ref, which may be stale' - ); - } -} - -/** Load every published export once; callers share the result. */ -function loadPublished(): PublishedFile[] { - return Object.values(PUBLISHED_EXPORTS).flatMap( - (f) => loadPublishedFile(f) ?? [] - ); -} - -// ---------- eval validation (fail before spending) ---------- - -function validateEvals(evalIds: string[]) { - for (const id of evalIds) { - const promptPath = join(ROOT, 'evals', id, 'PROMPT.md'); - if (!existsSync(promptPath)) - fail(`no eval at evals/${id} (PROMPT.md missing)`); - try { - parseEvalMarkdown( - readFileSync(promptPath, 'utf8'), - `evals/${id}/PROMPT.md` - ); - } catch (err) { - fail( - `eval metadata invalid — fix evals/${id}/PROMPT.md before spending on runs\n${err instanceof Error ? err.message : String(err)}` - ); - } - } -} - -function validateExperiment(experiment: string) { - if (!existsSync(join(ROOT, 'experiments', `${experiment}.ts`))) { - const available = readdirSync(join(ROOT, 'experiments')) - .filter((f) => f.endsWith('.ts')) - .map((f) => f.replace(/\.ts$/, '')); - fail( - `unknown experiment: ${experiment}\navailable: ${available.join(', ')}\n(or add experiments/${experiment}.ts — see any existing file for the shape)` - ); - } -} - -/** - * The agent itself needs its provider key. Checked here (not just by the - * harness) because the harness SKIPs the experiment with exit 0 on missing - * credentials — the runner would only notice at the no-result check. A - * set-but-EMPTY var counts as missing (node --env-file does not override - * an existing env var, even an empty one, so a stray `export KEY=` in the - * shell silently shadows .env — observed live). - */ -function validateAgentKey() { - if (process.env.ANTHROPIC_API_KEY) return; - fail( - process.env.ANTHROPIC_API_KEY === undefined - ? 'ANTHROPIC_API_KEY not set — add it to .env at the repo root' - : 'ANTHROPIC_API_KEY is set but EMPTY in your shell, which shadows .env (node --env-file never overrides an existing var) — unset it or export a real value' - ); -} - -/** - * Evals whose scorer uses the LLM judge grade the agent's output with an - * OpenAI model — even when the agent under test is Claude. A missing grader - * key otherwise surfaces only AFTER the (paid) agent run, wasting it. - * Textual scan of EVAL.ts; a false positive just asks for a key early. - */ -function validateJudgeKeys(evalIds: string[]) { - if (process.env.OPENAI_API_KEY) return; - const judged = evalIds.filter((id) => { - const scorer = join(ROOT, 'evals', id, 'EVAL.ts'); - return existsSync(scorer) && /\bjudge\b/.test(readFileSync(scorer, 'utf8')); - }); - if (judged.length) - fail( - `these evals score with the LLM judge (OpenAI-backed, regardless of the agent under test): ${judged.join(', ')}\nadd OPENAI_API_KEY to .env at the repo root before running them` - ); -} - -/** - * Accept either the mcp monorepo root or the server package dir for --mcp, - * and refuse pre-spend when the server isn't built (the harness would only - * discover that after eval setup). - */ -function resolveMcpServerPath(raw: string): string { - let p = isAbsolute(raw) ? raw : resolve(process.cwd(), raw); - if (!existsSync(p)) fail(`--mcp path does not exist: ${p}`); - const packageDir = join(p, 'packages', 'mcp-server-supabase'); - if (existsSync(packageDir)) p = packageDir; - if (!existsSync(join(p, 'dist', 'transports', 'stdio.js'))) - fail( - `no built server at ${p} (dist/transports/stdio.js missing) — build it first:\n pnpm install && pnpm build # in the mcp checkout (use \`mise exec --\` if corepack's pnpm mismatches)` - ); - // Fixture-drift heads-up: platform-lite tracks the pinned package version, - // and a local build from a newer line may call endpoints the fixture does - // not serve yet (observed: get_logs moved logs.all -> logs in 0.9.0 while - // the pin and fixture sat at 0.8.x). Warn, don't block. - try { - const local = JSON.parse( - readFileSync(join(p, 'package.json'), 'utf8') - ).version; - if (local && local !== MCP_SERVER_VERSION) - console.error( - `note: local mcp build is v${local}; the harness fixture (platform-lite) tracks the v${MCP_SERVER_VERSION} pin — endpoint drift is possible; judge by tool-call activation, not pass/fail alone` - ); - } catch { - /* unversioned checkout: nothing to compare */ - } - return p; -} - -/** - * The experiment's declared skills must exist in this checkout, or the - * treatment silently runs skill-less against a skills-enabled published - * baseline — a world mismatch, not a comparison. - */ -async function validateSkills(experiment: string) { - // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments) - const mod = await import( - pathToFileURL(join(ROOT, 'experiments', `${experiment}.ts`)).href - ); - const skills: string[] = (mod.default as ExperimentConfig).skills ?? []; - const missing = skills.filter((s) => !existsSync(join(ROOT, 'skills', s))); - if (missing.length) - fail( - `experiment ${experiment} declares skills this checkout is missing: ${missing.join(', ')}\ninitialise the skills submodule first: git submodule update --init` - ); -} - -// ---------- treatment run ---------- - -function runEval( - evalId: string, - experiment: string, - runs: number, - env: Record -): string { - const res = spawnSync( - process.execPath, - [ - '--import', - 'tsx/esm', - join(__dirname, '..', 'harness', 'run-eval.ts'), - '--eval', - evalId, - '--experiment', - experiment, - '--runs', - String(runs), - ], - { - stdio: 'inherit', - cwd: join(__dirname, '..'), - env: { ...process.env, ...env }, - } - ); - if (res.status !== 0) fail(`eval run failed: ${evalId} (exit ${res.status})`); - const resultPath = join(ROOT, 'results', experiment, `${evalId}.json`); - if (!existsSync(resultPath)) - fail( - `no result at results/${experiment}/${evalId}.json — check the eval/experiment ids` - ); - return resultPath; -} - -// test hook: LOCAL_EVAL_CMD writes the result file itself (no model spend) -function fakeRun(evalId: string, experiment: string): string { - const resultPath = join( - RESULTS_ROOT, - 'results', - experiment, - `${evalId}.json` - ); - mkdirSync(dirname(resultPath), { recursive: true }); - const res = spawnSync(process.env.LOCAL_EVAL_CMD as string, { - shell: true, - stdio: 'inherit', - env: { ...process.env, RES: resultPath, EVAL: evalId }, - }); - if (res.status !== 0) fail(`LOCAL_EVAL_CMD failed for ${evalId}`); - return resultPath; -} - -// ---------- reporting ---------- - -function reportRow( - label: string, - r: RawEvalResult | undefined, - extra: string -): string { - const checks = r?.checks ?? []; - const checksSummary = `${checks.filter((x) => x.passed).length}/${checks.length}`; - const docsCalls = r?.docs?.calls?.length ?? 0; - return `${label.padEnd(10)} passed=${String(r?.passed).padEnd(5)} checks=${checksSummary.padEnd(6)} docs.calls=${String(docsCalls).padEnd(3)} ${extra}`; -} - -/** Run one eval in the treatment world, write its receipt, report. */ -function runTreatment( - id: string, - experiment: string, - runs: number, - opts: { env: Record; mcpPath?: string } -): void { - const { env, mcpPath } = opts; - console.log( - `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}) ==` - ); - const resultPath = process.env.LOCAL_EVAL_CMD - ? fakeRun(id, experiment) - : runEval(id, experiment, runs, env); - - const parsed = rawEvalResultSchema.safeParse( - JSON.parse(readFileSync(resultPath, 'utf8')) - ); - if (!parsed.success) - fail( - `result at ${resultPath} does not match the eval result contract:\n${parsed.error.message}` - ); - const result = parsed.data; - const receipt = { - ...result, - provenance: collectProvenance(mcpPath), - }; - writeFileSync( - join(OUT_DIR, `${id}.treatment.json`), - `${JSON.stringify(receipt, null, 1)}\n` - ); - - console.log(`\n=== local run: ${id} (${experiment}) ===`); - console.log(reportRow('treatment', result, 'your world')); - console.log(`saved: results-local/${id}.treatment.json`); -} - -// ---------- subcommands ---------- - -async function cmdExperiments() { - const published = new Set( - loadPublished().flatMap((f) => f.rows.map((r) => r.experiment)) - ); - console.log( - `${'EXPERIMENT'.padEnd(36)} ${'AGENT'.padEnd(12)} ${'MODEL'.padEnd(22)} ${'EFFORT'.padEnd(8)} PUBLISHED` - ); - for (const f of readdirSync(join(ROOT, 'experiments')) - .filter((f) => f.endsWith('.ts')) - .sort()) { - const name = f.replace(/\.ts$/, ''); - // runtime-discovered plugin dir (same pattern as run-eval's loadExperiments) - const mod = await import(pathToFileURL(join(ROOT, 'experiments', f)).href); - const display = getExperimentDisplayMetadata( - mod.default as ExperimentConfig - ); - console.log( - `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes' : '-'}` - ); - } -} - -const RUN_USAGE = - 'usage: pnpm local run [--experiment ] [--runs N] [--mcp ]'; - -async function cmdRun(argv: string[]) { - const parsed = (() => { - try { - return parseArgs({ - args: argv, - options: { - experiment: { type: 'string' }, - runs: { type: 'string' }, - mcp: { type: 'string' }, - }, - allowPositionals: true, - }); - } catch (err) { - fail(`${err instanceof Error ? err.message : String(err)}\n${RUN_USAGE}`); - } - })(); - const { values, positionals } = parsed; - const experiment = values.experiment ?? DEFAULT_EXPERIMENT; - validateExperiment(experiment); - const evalIds = positionals; - if (!evalIds.length) fail(RUN_USAGE); - - validateEvals(evalIds); - // these gates are spend-relevant only for real runs; the test hook fakes them - if (!process.env.LOCAL_EVAL_CMD) { - validateAgentKey(); - await validateSkills(experiment); - validateJudgeKeys(evalIds); - } - - const env: Record = {}; - const mcpPath = values.mcp ? resolveMcpServerPath(values.mcp) : undefined; - if (mcpPath) env.SUPABASE_MCP_SERVER_PATH = mcpPath; - - mkdirSync(OUT_DIR, { recursive: true }); - for (const id of evalIds) { - const runs = Number(values.runs ?? 1); - runTreatment(id, experiment, runs, { env, mcpPath }); - } -} - -// ---------- entry ---------- - -const [command, ...rest] = process.argv.slice(2); -switch (command) { - case 'run': - await cmdRun(rest); - break; - case 'experiments': - await cmdExperiments(); - break; - default: - fail(`usage: pnpm local ... - run run eval(s) in your world (skills tree as-is; --mcp override) - experiments list experiments (agent, model, effort, published availability)`); -} diff --git a/apps/framework/scripts/published-log.ts b/apps/framework/scripts/published-log.ts deleted file mode 100644 index 8c895c2a..00000000 --- a/apps/framework/scripts/published-log.ts +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Parsing for the one-line `git log` record describing a published export's - * newest commit. Split out of local.ts so the smoke suite can exercise the - * merge-commit case directly: local.ts reads `origin/main`, whose history has - * no qualifying merge today, so an end-to-end check cannot reach it. - */ - -/** The `--format` string `parsePublishedLog` expects. Tab-separated on purpose. */ -export const PUBLISHED_LOG_FORMAT = '--format=%H%x09%P%x09%cI'; - -export type PublishedLog = { - /** Full sha of the newest commit touching the export. */ - commit: string; - /** First parent — the mainline, i.e. the ref the scheduled run built on. */ - parent: string; - /** Committer date, ISO 8601. */ - committedAt: string; -}; - -/** - * Parse one `PUBLISHED_LOG_FORMAT` line. - * - * Tab-separated because `%P` expands to EVERY parent, space-separated. Splitting - * the whole line on spaces therefore binds the second parent to `committedAt` on - * a merge commit, and a sha where an ISO date belongs poisons the baseline sort - * (`Date.parse` -> NaN) and prints "NaNd old" into the receipt. Path-limited log - * simplification hides most merges, but not one that changed the export relative - * to both parents — so this is reachable, not theoretical. - */ -export function parsePublishedLog(line: string): PublishedLog { - const bad = (why: string): never => { - throw new Error( - `unparseable published log line (${why}): ${JSON.stringify(line)}` - ); - }; - // Strip only the trailing newline. `trim()` would also eat a trailing empty - // field's separator, turning a malformed record into a confusing field count - // instead of a precise "committedAt is not a date". - const fields = line.replace(/\r?\n$/, '').split('\t'); - if (fields.length !== 3) - bad(`expected 3 tab-separated fields, got ${fields.length}`); - const [commit, parents, committedAt] = fields; - const isSha = (s: string) => /^[0-9a-f]{40}$/.test(s); - if (!isSha(commit)) bad('commit is not a sha'); - if (Number.isNaN(Date.parse(committedAt))) bad('committedAt is not a date'); - // A parentless commit means the premise behind `parent` (the ref the - // scheduled run built on) does not hold, so refuse rather than record ''. - const list = parents === '' ? [] : parents.split(' '); - if (!list.length) bad('commit has no parent'); - if (!list.every(isSha)) bad('parent is not a sha'); - return { commit, parent: list[0], committedAt }; -} diff --git a/package.json b/package.json index dbe8591f..ab7f505e 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,7 @@ "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp", "demo:executor": "pnpm --filter @supabase-evals/framework demo:executor", "format": "biome check --write . && pnpm --filter @supabase-evals/web format", - "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check", - "local": "pnpm --filter @supabase-evals/framework local" + "format:check": "biome check . && pnpm --filter @supabase-evals/web format:check" }, "dependencies": { "@ai-sdk/anthropic": "catalog:", From a500cd746bff449e1727e816e99120a530e4d231 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 11:11:18 +0200 Subject: [PATCH 04/12] test(eval): cover strict local runs --- apps/framework/scripts/smoke-local.ts | 435 ++++++++++++-------------- 1 file changed, 204 insertions(+), 231 deletions(-) diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts index 389cb118..94d6a5cd 100644 --- a/apps/framework/scripts/smoke-local.ts +++ b/apps/framework/scripts/smoke-local.ts @@ -1,284 +1,257 @@ -/** - * Zero-cost smoke test for the local-dev runner (scripts/local.ts). - * - * Fakes the eval run via LOCAL_EVAL_CMD (no model spend, no docker) and - * reads REAL published baselines from origin/main (no fetch: LOCAL_NO_FETCH). - * - * pnpm --filter @supabase-evals/framework test:local - */ import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, + readdirSync, rmSync, writeFileSync, } from 'node:fs'; -import { dirname, join } from 'node:path'; +import { join } from 'node:path'; import { tmpdir } from 'node:os'; -import { fileURLToPath } from 'node:url'; -import { parsePublishedLog, PUBLISHED_LOG_FORMAT } from './published-log.js'; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = join(__dirname, '..', '..', '..'); -// every output lands in a disposable sandbox — never the checkout's real -// results/ or results-local/ (an in-flight manual run may own those) -const SANDBOX = mkdtempSync(join(tmpdir(), 'smoke-local-')); -const OUT = join(SANDBOX, 'results-local'); +const ROOT = join(import.meta.dirname, '..', '..', '..'); +const SANDBOX = mkdtempSync(join(tmpdir(), 'smoke-eval-strict-')); const EXPERIMENT = 'claude-code-sonnet-5'; +const evalIds = readdirSync(join(ROOT, 'evals')).filter((id) => + existsSync(join(ROOT, 'evals', id, 'EVAL.ts')) +); +const EVAL = evalIds[0]; +assert.ok(EVAL, 'no eval fixture found'); +const JUDGED_EVAL = evalIds.find((id) => + /\bjudge\b/.test(readFileSync(join(ROOT, 'evals', id, 'EVAL.ts'), 'utf8')) +); +assert.ok(JUDGED_EVAL, 'no judged eval fixture found'); -// a published, currently-existing eval id — resolved dynamically so the test -// doesn't rot when the published set changes -const published = JSON.parse( - execFileSync( - 'git', - ['show', 'origin/main:apps/web/src/data/regression-eval-results.json'], - { cwd: ROOT, maxBuffer: 1 << 28 } - ).toString() -) as Array<{ experiment: string; eval: string }>; -const EVAL = published.find( - (r) => r.experiment === EXPERIMENT && existsSync(join(ROOT, 'evals', r.eval)) -)?.eval; -assert.ok(EVAL, 'no published eval with a local evals/ dir found'); - -// LOCAL_EVAL_CMD contract: write a result JSON to $RES for eval $EVAL. -// A script file sidesteps per-platform shell quoting entirely. const fakeScript = join(SANDBOX, 'fake-eval.cjs'); writeFileSync( fakeScript, `const fs = require('node:fs'); const path = require('node:path'); fs.mkdirSync(path.dirname(process.env.RES), { recursive: true }); -fs.writeFileSync( - process.env.RES, - JSON.stringify({ - eval: process.env.EVAL, - experiment: '${EXPERIMENT}', - passed: true, - checks: [{ name: 'x', passed: true }], - }) -); +fs.writeFileSync(process.env.RES, JSON.stringify({ + passed: true, + checks: [{ name: 'fake run', passed: true }], + attempts: 1, + skills: { available: [], loaded: [] }, + docs: { calls: [] }, + toolCalls: [], + transcript: [], + agentReport: '', + stoppedReason: 'end_turn' +})); ` ); const FAKE = `${JSON.stringify(process.execPath)} ${JSON.stringify(fakeScript)}`; -function local(args: string[], env: Record = {}) { - const res = spawnSync( - process.execPath, - ['--import', 'tsx/esm', join(__dirname, 'local.ts'), ...args], - { - cwd: join(__dirname, '..'), - encoding: 'utf8', - timeout: 60_000, // a regressed pre-spend gate must never reach a real agent run - env: { - ...process.env, - LOCAL_NO_FETCH: '1', - LOCAL_RESULTS_ROOT: SANDBOX, - LOCAL_EVAL_CMD: FAKE, - FORCE_COLOR: '0', - ...env, - }, - } - ); - return { out: `${res.stdout}\n${res.stderr}`, status: res.status }; +function runEval(args: string[], env: Record = {}) { + const result = spawnSync('pnpm', ['eval', '--', ...args], { + cwd: ROOT, + encoding: 'utf8', + timeout: 60_000, + env: { + ...process.env, + ANTHROPIC_API_KEY: 'placeholder', + OPENAI_API_KEY: 'placeholder', + LOCAL_EVAL_CMD: FAKE, + LOCAL_RESULTS_ROOT: SANDBOX, + FORCE_COLOR: '0', + ...env, + }, + }); + return { + output: `${result.stdout}\n${result.stderr}`, + status: result.status, + }; } let passed = 0; -function ck(name: string, fn: () => void) { +function check(name: string, assertion: () => void) { try { - fn(); - passed++; - } catch (err) { + assertion(); + passed += 1; + } catch (error) { console.error(`FAIL: ${name}`); - throw err; + throw error; } } -// --- published-log parsing: the merge-commit case origin/main cannot reach --- -// Built here rather than asserted against real history: main has no merge that -// touches a published export, so an end-to-end check would pass either way. -{ - const repo = join(SANDBOX, 'merge-parse-repo'); - const file = 'exports.json'; - const g = (args: string[]) => - execFileSync('git', args, { cwd: repo, encoding: 'utf8' }).trim(); - mkdirSync(repo, { recursive: true }); - g(['init', '-q', '-b', 'main']); - g(['config', 'user.email', 'smoke@local']); - g(['config', 'user.name', 'smoke']); - const commitFile = (body: string, msg: string) => { - writeFileSync(join(repo, file), body); - g(['add', file]); - g(['commit', '-qm', msg]); - }; - commitFile('[{"eval":"base"}]\n', 'base'); - g(['checkout', '-q', '-b', 'side']); - commitFile('[{"eval":"side"}]\n', 'side'); - g(['checkout', '-q', 'main']); - commitFile('[{"eval":"main"}]\n', 'main edit'); - // Expected to conflict: we resolve to content differing from BOTH parents so - // path-limited history simplification keeps this merge in `git log -- `. - // The conflict is the point, so assert a merge is genuinely in progress rather - // than discarding the exit status and hoping. - const merge = spawnSync('git', ['merge', 'side'], { - cwd: repo, - encoding: 'utf8', - }); - assert.notEqual( - merge.status, - 0, - `expected \`git merge side\` to conflict, got clean exit: ${merge.stdout}${merge.stderr}` - ); - assert.ok( - existsSync(join(repo, '.git', 'MERGE_HEAD')), - 'no MERGE_HEAD: the next commit would not be a merge commit' - ); - commitFile('[{"eval":"merged"}]\n', 'merge side into main'); - - const line = g(['log', 'main', '-1', PUBLISHED_LOG_FORMAT, '--', file]); +try { + { + const result = runEval([ + '--strict', + '--experiment', + 'bogus-model', + '--eval', + EVAL, + ]); + check('unknown experiment is refused', () => { + assert.equal(result.status, 1); + assert.match(result.output, /no experiment matched: bogus-model/); + }); + } - ck('published log line for a merge commit really has 2 parents', () => { - const [, parents] = line.split('\t'); - assert.equal(parents.split(' ').length, 2, `expected a merge: ${line}`); - }); + { + const result = runEval([ + '--strict', + '--experiment', + EXPERIMENT, + '--eval', + 'not-an-eval-dir', + ]); + check('unknown eval is refused', () => { + assert.equal(result.status, 1); + assert.match(result.output, /no eval matched: not-an-eval-dir/); + }); + } - ck('merge commit parses to the mainline parent and a real date', () => { - const parsed = parsePublishedLog(line); - assert.match(parsed.commit, /^[0-9a-f]{40}$/); - assert.match(parsed.parent, /^[0-9a-f]{40}$/); - assert.equal(parsed.parent, g(['rev-parse', 'main^1'])); - assert.ok( - !Number.isNaN(Date.parse(parsed.committedAt)), - `committedAt is not a date: ${parsed.committedAt}` + { + const result = runEval( + ['--strict', '--experiment', EXPERIMENT, '--eval', EVAL], + { ANTHROPIC_API_KEY: '' } ); - }); + check('strict refuses a missing agent key', () => { + assert.equal(result.status, 1); + assert.match(result.output, /ANTHROPIC_API_KEY/); + }); + } - ck('the old space-split is what this guards against', () => { - // Reproduce the pre-fix parse to prove the regression is real: `%P` puts a - // second sha where the timestamp belongs, and Date.parse yields NaN. - const spaceSplit = g([ - 'log', - 'main', - '-1', - '--format=%H %P %cI', - '--', - file, - ]).split(' '); - assert.equal(spaceSplit.length, 4); - assert.ok(Number.isNaN(Date.parse(spaceSplit[2]))); - }); + { + const result = runEval(['--experiment', EXPERIMENT, '--eval', EVAL], { + ANTHROPIC_API_KEY: '', + }); + check('default mode keeps the missing-key skip', () => { + assert.equal(result.status, 0); + assert.match(result.output, new RegExp(`SKIP ${EXPERIMENT}`)); + }); + } - ck('malformed published log lines are refused, not guessed', () => { - assert.throws(() => parsePublishedLog('only-one-field'), /expected 3/); - assert.throws( - () => parsePublishedLog('nothex\tdead\t2026-01-01'), - /not a sha/ - ); - assert.throws( - () => parsePublishedLog(`${'a'.repeat(40)}\t${'b'.repeat(40)}\tnope`), - /not a date/ + { + const emptySkills = join(SANDBOX, 'empty-skills'); + mkdirSync(emptySkills); + const result = runEval( + ['--strict', '--experiment', EXPERIMENT, '--eval', EVAL], + { LOCAL_SKILLS_ROOT: emptySkills } ); - assert.throws( - () => parsePublishedLog(`${'a'.repeat(40)}\t\t2026-01-01T00:00:00Z`), - /no parent/ + check('strict refuses missing experiment skills', () => { + assert.equal(result.status, 1); + assert.match(result.output, /declares skills this checkout is missing/); + assert.match(result.output, /git submodule update --init/); + }); + } + + { + const result = runEval( + ['--strict', '--experiment', EXPERIMENT, '--eval', JUDGED_EVAL], + { OPENAI_API_KEY: '', LOCAL_EVAL_CMD: '' } ); - }); -} + check('judged eval without OPENAI_API_KEY is refused pre-spend', () => { + assert.equal(result.status, 1); + assert.match(result.output, /score with the LLM judge/); + assert.match(result.output, /add OPENAI_API_KEY/); + }); + } -// --- refusals happen pre-spend, with actionable messages --- -{ - const r = local(['run', EVAL, '--experiment', 'bogus-model']); - ck('unknown experiment refused with the available list', () => { - assert.equal(r.status, 1); - assert.match(r.out, /unknown experiment: bogus-model/); - assert.match(r.out, /claude-code-sonnet-5/); - }); -} -{ - const r = local(['run', 'not-an-eval-dir']); - ck('missing eval dir refused', () => { - assert.equal(r.status, 1); - assert.match(r.out, /no eval at evals\/not-an-eval-dir/); - }); -} + { + const result = runEval([ + '--strict', + '--experiment', + EXPERIMENT, + '--eval', + EVAL, + '--mcp', + '/definitely/not/a/path', + ]); + check('missing MCP override path is refused', () => { + assert.equal(result.status, 1); + assert.match(result.output, /--mcp path does not exist/); + }); + } -// --- run: no baseline required (custom evals), receipt only --- -{ - const r = local(['run', EVAL]); - ck('run works without published baseline machinery', () => { - assert.equal(r.status, 0); - assert.match(r.out, new RegExp(`=== local run: ${EVAL}`)); - assert.doesNotMatch(r.out, /published /); - assert.match(r.out, /saved: results-local\//); - }); -} + const mcpCheckout = join(SANDBOX, 'mcp-checkout'); + const mcpPackage = join(mcpCheckout, 'packages', 'mcp-server-supabase'); + mkdirSync(mcpPackage, { recursive: true }); + writeFileSync(join(mcpPackage, 'package.json'), '{"version":"0.0.0"}'); -// --- mcp override path validation --- -{ - const r = local(['run', EVAL, '--mcp', '/definitely/not/a/path']); - ck('bad --mcp path refused pre-spend', () => { - assert.equal(r.status, 1); - assert.match(r.out, /--mcp path does not exist/); - }); -} + { + const result = runEval([ + '--strict', + '--experiment', + EXPERIMENT, + '--eval', + EVAL, + '--mcp', + mcpCheckout, + ]); + check('unbuilt MCP override is refused with a build hint', () => { + assert.equal(result.status, 1); + assert.match(result.output, /no built server at .*mcp-server-supabase/); + assert.match(result.output, /pnpm install && pnpm build/); + }); + } -// --- mcp override: monorepo root resolves to the server package; unbuilt refused --- -{ - const fake = join(SANDBOX, '.smoke-mcp-checkout'); - const pkg = join(fake, 'packages', 'mcp-server-supabase'); - mkdirSync(join(pkg, 'dist', 'transports'), { recursive: true }); + mkdirSync(join(mcpPackage, 'dist', 'transports'), { recursive: true }); + writeFileSync(join(mcpPackage, 'dist', 'transports', 'stdio.js'), ''); - const unbuilt = local(['run', EVAL, '--mcp', fake]); - ck('unbuilt mcp checkout refused pre-spend with build hint', () => { - assert.equal(unbuilt.status, 1); - assert.match(unbuilt.out, /no built server at .*mcp-server-supabase/); - assert.match(unbuilt.out, /pnpm install && pnpm build/); - }); + { + const result = runEval([ + '--strict', + '--experiment', + EXPERIMENT, + '--eval', + EVAL, + '--mcp', + mcpCheckout, + ]); + check('built MCP override reaches the eval path', () => { + assert.equal(result.status, 0); + assert.match(result.output, new RegExp(`PASS ${EXPERIMENT} x ${EVAL}`)); + }); + } - writeFileSync( - join(pkg, 'dist', 'transports', 'stdio.js'), - '// smoke fixture\n' - ); - const built = local(['run', EVAL, '--mcp', fake]); - ck('monorepo root resolves to the server package dir', () => { - assert.equal(built.status, 0); - const receipt = JSON.parse( - readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8') - ); + const resultPath = join(SANDBOX, 'results', EXPERIMENT, `${EVAL}.json`); + const receipt = JSON.parse(readFileSync(resultPath, 'utf8')); + check('result receipt stays under results experiment subdirectory', () => { + assert.equal(receipt.eval, EVAL); + assert.equal(receipt.experiment, EXPERIMENT); + assert.ok(receipt.provenance.generatedAt); + assert.equal(receipt.provenance.host.sha.length, 40); assert.match( receipt.provenance.mcpOverride.path, /packages[/\\]mcp-server-supabase$/ ); + assert.equal(existsSync(join(SANDBOX, 'results-local')), false); }); - rmSync(fake, { recursive: true, force: true }); -} -// --- judge-key gate: refused pre-spend, before any agent spawn --- -{ - // needs an eval whose scorer really uses the judge; EVAL may not - const judgedEval = published.find( - (row) => - row.experiment === EXPERIMENT && - existsSync(join(ROOT, 'evals', row.eval, 'EVAL.ts')) && - /\bjudge\b/.test( - readFileSync(join(ROOT, 'evals', row.eval, 'EVAL.ts'), 'utf8') - ) - )?.eval; - assert.ok(judgedEval, 'no judged eval found in the published set'); - const r = local(['run', judgedEval], { - LOCAL_EVAL_CMD: '', - OPENAI_API_KEY: '', - }); - ck('judged eval without OPENAI_API_KEY refused pre-spend', () => { - assert.equal(r.status, 1); - assert.match(r.out, /score with the LLM judge/); - assert.match(r.out, /add OPENAI_API_KEY/); - }); -} + { + const result = runEval([ + '--strict', + '--skip-existing', + '--experiment', + EXPERIMENT, + '--eval', + EVAL, + ]); + check('strict keeps skip-existing intentional', () => { + assert.equal(result.status, 0); + assert.match(result.output, /already ran/); + }); + } -// cleanup: everything lived in the sandbox -rmSync(SANDBOX, { recursive: true, force: true }); + { + const result = runEval(['list', '--strict', '--eval', EVAL], { + ANTHROPIC_API_KEY: '', + OPENAI_API_KEY: '', + }); + check('strict keeps list planning free of credential gates', () => { + assert.equal(result.status, 0); + assert.match(result.output, /claude-code-sonnet-5/); + }); + } +} finally { + rmSync(SANDBOX, { recursive: true, force: true }); +} console.log(`smoke-local: ${passed} checks passed`); From 460499570583b90455e0d25bfc35fab374ccf958 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 11:11:35 +0200 Subject: [PATCH 05/12] ci(eval): enable strict runner mode --- .github/workflows/eval-refresh.yml | 1 + apps/framework/scripts/run-vercel-evals.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 413060f2..d6dcdc4d 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -309,6 +309,7 @@ jobs: set -euo pipefail pnpm --filter @supabase-evals/framework eval:vercel -- \ + --strict \ --pairs-json "$EVAL_PAIRS" \ --revision "$EVAL_REVISION" \ --runs "${{ needs.prepare.outputs.runs }}" \ diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 0f21eb0f..5d01e822 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -51,6 +51,7 @@ interface RunnerOptions { timeoutSec: number; concurrency: number; vcpus: number; + strict: boolean; } interface PairOptions extends RunnerOptions { @@ -255,6 +256,7 @@ async function runPairOnce( args: [ 'eval', '--', + ...(options.strict ? ['--strict'] : []), '--experiment', pair.experiment, '--experiment-suite', @@ -622,6 +624,7 @@ async function main(): Promise { 'concurrency' ), vcpus: positiveInteger(readFlag(rawArgs, 'vcpus') ?? '4', 'vcpus'), + strict: rawArgs.includes('--strict'), }; console.log( From 96448278163a6536fbd0af552bd36c9ebdd9a499 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 11:16:04 +0200 Subject: [PATCH 06/12] fix(eval): skip unavailable published pairs --- .../scripts/run-vercel-evals.test.ts | 34 +++++++++++++++++++ apps/framework/scripts/run-vercel-evals.ts | 28 +++++++++++++-- apps/framework/scripts/smoke-local.ts | 22 ++++++------ 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/apps/framework/scripts/run-vercel-evals.test.ts b/apps/framework/scripts/run-vercel-evals.test.ts index 538c4407..c0029738 100644 --- a/apps/framework/scripts/run-vercel-evals.test.ts +++ b/apps/framework/scripts/run-vercel-evals.test.ts @@ -1,6 +1,10 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { APIError } from '@vercel/sandbox'; import { describe, expect, it } from 'vitest'; import { + filterLocallyAvailablePairs, isRetryableSandboxCreateError, isTerminalSandboxCreateError, parsePairs, @@ -56,6 +60,36 @@ describe('Vercel eval controller', () => { ); }); + it('skips published evals absent from the checked-out tree', () => { + const evalsRoot = mkdtempSync(join(tmpdir(), 'eval-pairs-')); + const localEval = join(evalsRoot, 'local-eval'); + mkdirSync(localEval); + writeFileSync(join(localEval, 'PROMPT.md'), 'malformed on purpose'); + const pair = { + experiment: 'experiment-1', + experiment_suite: 'benchmark', + eval_suite: 'benchmark', + }; + const notes: string[] = []; + try { + expect( + filterLocallyAvailablePairs( + [ + { ...pair, eval_id: 'local-eval' }, + { ...pair, eval_id: 'newer-main-eval' }, + ], + evalsRoot, + (message) => notes.push(message) + ) + ).toEqual([{ ...pair, eval_id: 'local-eval' }]); + expect(notes).toEqual([ + 'SKIP newer-main-eval (published eval is absent from this checkout: evals/newer-main-eval/PROMPT.md not found)', + ]); + } finally { + rmSync(evalsRoot, { recursive: true, force: true }); + } + }); + it('retries sandbox creation only on 429s and 5xx API responses', () => { const apiError = (status: number) => new APIError(new Response(null, { status })); diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index 5d01e822..d7e6b58b 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -2,7 +2,7 @@ import { APIError, Sandbox } from '@vercel/sandbox'; import { execFile, execFileSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; +import { existsSync, mkdtempSync, mkdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -505,6 +505,29 @@ export function parsePairs(value: string): EvalPair[] { return parsed.data; } +/** + * Drops pairs published by a newer main revision when this checkout does not + * contain their prompt. Existing local evals still reach run-eval validation. + */ +export function filterLocallyAvailablePairs( + pairs: readonly EvalPair[], + evalsRoot = join(ROOT, 'evals'), + note: (message: string) => void = console.warn +): EvalPair[] { + const missing = new Set(); + const available = pairs.filter((pair) => { + if (existsSync(join(evalsRoot, pair.eval_id, 'PROMPT.md'))) return true; + missing.add(pair.eval_id); + return false; + }); + for (const evalId of missing) { + note( + `SKIP ${evalId} (published eval is absent from this checkout: evals/${evalId}/PROMPT.md not found)` + ); + } + return available; +} + /** Returns an environment variable or a useful configuration error. */ function requireEnv(name: string, hint: string): string { const value = process.env[name]; @@ -605,9 +628,8 @@ async function main(): Promise { const rawArgs = process.argv.slice(2).filter((arg) => arg !== '--'); const pairsValue = readFlag(rawArgs, 'pairs-json') ?? process.env.EVAL_PAIRS; if (!pairsValue) throw new Error('--pairs-json or EVAL_PAIRS is required'); - const options: RunnerOptions = { - pairs: parsePairs(pairsValue), + pairs: filterLocallyAvailablePairs(parsePairs(pairsValue)), revision: readFlag(rawArgs, 'revision') ?? currentRevision(), repoUrl: readFlag(rawArgs, 'repo-url') ?? repositoryUrl(), outputDir: resolve( diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts index 94d6a5cd..969db230 100644 --- a/apps/framework/scripts/smoke-local.ts +++ b/apps/framework/scripts/smoke-local.ts @@ -88,7 +88,7 @@ try { EVAL, ]); check('unknown experiment is refused', () => { - assert.equal(result.status, 1); + assert.equal(result.status, 1, result.output); assert.match(result.output, /no experiment matched: bogus-model/); }); } @@ -102,7 +102,7 @@ try { 'not-an-eval-dir', ]); check('unknown eval is refused', () => { - assert.equal(result.status, 1); + assert.equal(result.status, 1, result.output); assert.match(result.output, /no eval matched: not-an-eval-dir/); }); } @@ -113,7 +113,7 @@ try { { ANTHROPIC_API_KEY: '' } ); check('strict refuses a missing agent key', () => { - assert.equal(result.status, 1); + assert.equal(result.status, 1, result.output); assert.match(result.output, /ANTHROPIC_API_KEY/); }); } @@ -123,7 +123,7 @@ try { ANTHROPIC_API_KEY: '', }); check('default mode keeps the missing-key skip', () => { - assert.equal(result.status, 0); + assert.equal(result.status, 0, result.output); assert.match(result.output, new RegExp(`SKIP ${EXPERIMENT}`)); }); } @@ -136,7 +136,7 @@ try { { LOCAL_SKILLS_ROOT: emptySkills } ); check('strict refuses missing experiment skills', () => { - assert.equal(result.status, 1); + assert.equal(result.status, 1, result.output); assert.match(result.output, /declares skills this checkout is missing/); assert.match(result.output, /git submodule update --init/); }); @@ -148,7 +148,7 @@ try { { OPENAI_API_KEY: '', LOCAL_EVAL_CMD: '' } ); check('judged eval without OPENAI_API_KEY is refused pre-spend', () => { - assert.equal(result.status, 1); + assert.equal(result.status, 1, result.output); assert.match(result.output, /score with the LLM judge/); assert.match(result.output, /add OPENAI_API_KEY/); }); @@ -165,7 +165,7 @@ try { '/definitely/not/a/path', ]); check('missing MCP override path is refused', () => { - assert.equal(result.status, 1); + assert.equal(result.status, 1, result.output); assert.match(result.output, /--mcp path does not exist/); }); } @@ -186,7 +186,7 @@ try { mcpCheckout, ]); check('unbuilt MCP override is refused with a build hint', () => { - assert.equal(result.status, 1); + assert.equal(result.status, 1, result.output); assert.match(result.output, /no built server at .*mcp-server-supabase/); assert.match(result.output, /pnpm install && pnpm build/); }); @@ -206,7 +206,7 @@ try { mcpCheckout, ]); check('built MCP override reaches the eval path', () => { - assert.equal(result.status, 0); + assert.equal(result.status, 0, result.output); assert.match(result.output, new RegExp(`PASS ${EXPERIMENT} x ${EVAL}`)); }); } @@ -235,7 +235,7 @@ try { EVAL, ]); check('strict keeps skip-existing intentional', () => { - assert.equal(result.status, 0); + assert.equal(result.status, 0, result.output); assert.match(result.output, /already ran/); }); } @@ -246,7 +246,7 @@ try { OPENAI_API_KEY: '', }); check('strict keeps list planning free of credential gates', () => { - assert.equal(result.status, 0); + assert.equal(result.status, 0, result.output); assert.match(result.output, /claude-code-sonnet-5/); }); } From 887a8dedd09556b300c6dac47a31bad8a6cdfe02 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 11:29:35 +0200 Subject: [PATCH 07/12] fix(eval): validate local planning inputs --- .github/workflows/eval-refresh.yml | 22 +++++++++++++++++++--- apps/framework/scripts/smoke-local.ts | 13 ++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index d6dcdc4d..47cacb7b 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -174,7 +174,12 @@ jobs: if [ -n "$eval_ids" ]; then matching=() while IFS= read -r id; do - [ -d "evals/$id" ] && matching+=("$id") + if [ -f "evals/$id/PROMPT.md" ]; then + pnpm --silent eval -- list --eval "$id" > /dev/null + matching+=("$id") + else + echo "SKIP $id (published eval is absent from this checkout: evals/$id/PROMPT.md not found)" + fi done < <(jq -Rr 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | .[]' <<< "$eval_ids") else matching=() @@ -182,7 +187,11 @@ jobs: [ -d "$dir" ] || continue id=$(basename "$dir") prompt="$dir/PROMPT.md" - [ -f "$prompt" ] || continue + if [ ! -f "$prompt" ]; then + echo "SKIP $id (eval directory has no PROMPT.md)" + continue + fi + pnpm --silent eval -- list --eval "$id" > /dev/null suite_val=$(sed -n 's/^suite:[[:space:]]*//p' "$prompt" | head -n 1) if jq -e --arg s "$suite_val" 'index($s) != null' <<< "$suite_json" > /dev/null 2>&1; then matching+=("$id") @@ -223,7 +232,14 @@ jobs: case "$eval_suite" in benchmark) experiment_suites=(benchmark no-skills) ;; regression) experiment_suites=(regression) ;; - *) continue ;; + other) + echo "SKIP $id (suite other has no scheduled experiment suite)" + continue + ;; + *) + echo "Invalid suite in evals/$id/PROMPT.md: $eval_suite" >&2 + exit 1 + ;; esac for experiment_suite in "${experiment_suites[@]}"; do diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts index 969db230..18ff23d2 100644 --- a/apps/framework/scripts/smoke-local.ts +++ b/apps/framework/scripts/smoke-local.ts @@ -15,9 +15,16 @@ import { tmpdir } from 'node:os'; const ROOT = join(import.meta.dirname, '..', '..', '..'); const SANDBOX = mkdtempSync(join(tmpdir(), 'smoke-eval-strict-')); const EXPERIMENT = 'claude-code-sonnet-5'; -const evalIds = readdirSync(join(ROOT, 'evals')).filter((id) => - existsSync(join(ROOT, 'evals', id, 'EVAL.ts')) -); +const evalIds = readdirSync(join(ROOT, 'evals')).filter((id) => { + const prompt = join(ROOT, 'evals', id, 'PROMPT.md'); + if (!existsSync(prompt)) { + console.warn( + `SKIP ${id} (eval is absent from this checkout: evals/${id}/PROMPT.md not found)` + ); + return false; + } + return existsSync(join(ROOT, 'evals', id, 'EVAL.ts')); +}); const EVAL = evalIds[0]; assert.ok(EVAL, 'no eval fixture found'); const JUDGED_EVAL = evalIds.find((id) => From 8ce6e4f87d0673b8b21063c4661bcf91395c5cfb Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 11:35:53 +0200 Subject: [PATCH 08/12] fix(eval): allow empty published plans --- apps/framework/scripts/run-vercel-evals.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/framework/scripts/run-vercel-evals.ts b/apps/framework/scripts/run-vercel-evals.ts index d7e6b58b..1de23125 100644 --- a/apps/framework/scripts/run-vercel-evals.ts +++ b/apps/framework/scripts/run-vercel-evals.ts @@ -654,6 +654,7 @@ async function main(): Promise { ); for (const pair of options.pairs) console.log(`PLAN ${pairLabel(pair)}`); if (rawArgs.includes('--dry-run')) return; + if (options.pairs.length === 0) return; await runPairs(options); } From 7f6689b4c9b4c5430c0c33902c2041b60d1f0fc7 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 12:39:09 +0200 Subject: [PATCH 09/12] fix(eval): harden strict result handling --- apps/framework/harness/run-eval.ts | 61 ++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index ae81d246..e79595ba 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -7,6 +7,7 @@ import { readdirSync, readFileSync, realpathSync, + renameSync, rmSync, statSync, writeFileSync, @@ -15,6 +16,7 @@ import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { jsonSchema, tool, type ToolSet } from 'ai'; import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; +import { rawEvalResultSchema } from '@supabase-evals/core/eval-metadata'; import { createBareSandbox, frontmatterDescription, @@ -723,7 +725,7 @@ function validateStrictSkills( ) { if (!STRICT) return; const missing = skillNames.filter( - (name) => !existsSync(join(SKILLS_ROOT, name)) + (name) => !existsSync(join(SKILLS_ROOT, name, 'SKILL.md')) ); if (missing.length > 0) { throw new Error( @@ -731,20 +733,25 @@ function validateStrictSkills( ); } } - function fakeRun(out: string, experiment: string, evalId: string): RunResult { const command = process.env.LOCAL_EVAL_CMD; if (!command) throw new Error('LOCAL_EVAL_CMD is required'); mkdirSync(dirname(out), { recursive: true }); + const fakeOut = `${out}.fake`; const result = spawnSync(command, { shell: true, stdio: 'inherit', - env: { ...process.env, RES: out, EVAL: evalId, EXPERIMENT: experiment }, + env: { ...process.env, RES: fakeOut, EVAL: evalId, EXPERIMENT: experiment }, }); if (result.status !== 0) { + rmSync(fakeOut, { force: true }); throw new Error(`LOCAL_EVAL_CMD failed for ${experiment} x ${evalId}`); } - return JSON.parse(readFileSync(out, 'utf8')) as RunResult; + try { + return JSON.parse(readFileSync(fakeOut, 'utf8')) as RunResult; + } finally { + rmSync(fakeOut, { force: true }); + } } async function main() { @@ -871,8 +878,19 @@ async function main() { for (const ev of suiteFiltered) { const out = resultPath(name, ev); if (!FORCE && existsSync(out)) { - console.log(`SKIP ${name} x ${ev.id} (already ran)`); - continue; + let existingResultIsValid = false; + try { + existingResultIsValid = rawEvalResultSchema.safeParse( + JSON.parse(readFileSync(out, 'utf8')) + ).success; + } catch { + // A partial write is incomplete work and must run again. + } + if (existingResultIsValid) { + console.log(`SKIP ${name} x ${ev.id} (already ran)`); + continue; + } + console.log(`RERUN ${name} x ${ev.id} (existing result is invalid)`); } if (ev.mode === 'local-stack' && !config.localStack) { const message = @@ -898,6 +916,7 @@ async function main() { validateJudgeKeys([...new Set(allWork.map(({ ev }) => ev))]); if (!DEBUG) console.error = () => undefined; + const provenance = collectProvenance(mcpPath); let localStackTurn = Promise.resolve(); const errored: Error[] = []; @@ -913,22 +932,22 @@ async function main() { : await runOne(name, config, ev); mkdirSync(dirname(out), { recursive: true }); const experimentDisplay = getExperimentDisplayMetadata(config); - writeFileSync( - out, - JSON.stringify( - { - experiment: name, - experimentSuite: SELECTED_EXPERIMENT_SUITE ?? config.suite?.[0], - experimentDisplay, - eval: ev.id, - ...ev.metadata, - ...res, - provenance: collectProvenance(mcpPath), - }, - null, - 2 - ) + const resultJson = JSON.stringify( + { + experiment: name, + experimentSuite: SELECTED_EXPERIMENT_SUITE ?? config.suite?.[0], + experimentDisplay, + eval: ev.id, + ...ev.metadata, + ...res, + provenance, + }, + null, + 2 ); + const temporaryOut = `${out}.tmp`; + writeFileSync(temporaryOut, resultJson); + renameSync(temporaryOut, out); const elapsed = Math.round((Date.now() - start) / 1000); console.log( `${res.passed ? '✅ PASS' : '❌ FAIL'} ${name} x ${ev.id} (${formatRunSummary(res)}, ${elapsed}s)\n → ${relative(ROOT, out)}` From ef04e57d4b016a3e1b773958a41b3b988bad1eae Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 12:39:20 +0200 Subject: [PATCH 10/12] test(eval): cover strict result boundaries --- apps/framework/scripts/smoke-local.ts | 77 ++++++++++++++++++++------- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts index 18ff23d2..b5a0c3f9 100644 --- a/apps/framework/scripts/smoke-local.ts +++ b/apps/framework/scripts/smoke-local.ts @@ -75,12 +75,17 @@ function runEval(args: string[], env: Record = {}) { } let passed = 0; -function check(name: string, assertion: () => void) { +function check(name: string, assertion: () => void, diagnostic?: string) { try { assertion(); passed += 1; } catch (error) { console.error(`FAIL: ${name}`); + if (diagnostic) { + throw new Error(`${name} failed\n\nChild output:\n${diagnostic}`, { + cause: error, + }); + } throw error; } } @@ -149,6 +154,22 @@ try { }); } + { + const partialSkills = join(SANDBOX, 'partial-skills'); + mkdirSync(join(partialSkills, 'supabase'), { recursive: true }); + mkdirSync(join(partialSkills, 'supabase-postgres-best-practices'), { + recursive: true, + }); + const result = runEval( + ['--strict', '--experiment', EXPERIMENT, '--eval', EVAL], + { LOCAL_SKILLS_ROOT: partialSkills } + ); + check('strict refuses skill directories without SKILL.md', () => { + assert.equal(result.status, 1, result.output); + assert.match(result.output, /declares skills this checkout is missing/); + }); + } + { const result = runEval( ['--strict', '--experiment', EXPERIMENT, '--eval', JUDGED_EVAL], @@ -202,36 +223,56 @@ try { mkdirSync(join(mcpPackage, 'dist', 'transports'), { recursive: true }); writeFileSync(join(mcpPackage, 'dist', 'transports', 'stdio.js'), ''); + const receiptRun = runEval([ + '--strict', + '--experiment', + EXPERIMENT, + '--eval', + EVAL, + '--mcp', + mcpCheckout, + ]); + check('built MCP override reaches the eval path', () => { + assert.equal(receiptRun.status, 0, receiptRun.output); + assert.match(receiptRun.output, new RegExp(`PASS ${EXPERIMENT} x ${EVAL}`)); + }); + + const resultPath = join(SANDBOX, 'results', EXPERIMENT, `${EVAL}.json`); + check( + 'result receipt stays under results experiment subdirectory', + () => { + const receipt = JSON.parse(readFileSync(resultPath, 'utf8')); + assert.equal(receipt.eval, EVAL); + assert.equal(receipt.experiment, EXPERIMENT); + assert.ok(receipt.provenance.generatedAt); + assert.equal(receipt.provenance.host.sha.length, 40); + assert.match( + receipt.provenance.mcpOverride.path, + /packages[/\\]mcp-server-supabase$/ + ); + assert.equal(existsSync(join(SANDBOX, 'results-local')), false); + }, + receiptRun.output + ); + { + writeFileSync(resultPath, '{'); const result = runEval([ '--strict', + '--skip-existing', '--experiment', EXPERIMENT, '--eval', EVAL, - '--mcp', - mcpCheckout, ]); - check('built MCP override reaches the eval path', () => { + check('skip-existing reruns a corrupt result', () => { assert.equal(result.status, 0, result.output); + assert.match(result.output, /existing result is invalid/); assert.match(result.output, new RegExp(`PASS ${EXPERIMENT} x ${EVAL}`)); + assert.doesNotThrow(() => JSON.parse(readFileSync(resultPath, 'utf8'))); }); } - const resultPath = join(SANDBOX, 'results', EXPERIMENT, `${EVAL}.json`); - const receipt = JSON.parse(readFileSync(resultPath, 'utf8')); - check('result receipt stays under results experiment subdirectory', () => { - assert.equal(receipt.eval, EVAL); - assert.equal(receipt.experiment, EXPERIMENT); - assert.ok(receipt.provenance.generatedAt); - assert.equal(receipt.provenance.host.sha.length, 40); - assert.match( - receipt.provenance.mcpOverride.path, - /packages[/\\]mcp-server-supabase$/ - ); - assert.equal(existsSync(join(SANDBOX, 'results-local')), false); - }); - { const result = runEval([ '--strict', From a61e6f1181de42adecdef70dd6cde3007e0b50f1 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 14:00:19 +0200 Subject: [PATCH 11/12] fix(eval): reject unknown CLI arguments --- apps/framework/harness/run-eval.ts | 30 +++++++++++ apps/framework/lib/cli-args.test.ts | 35 ++++++++++++- apps/framework/lib/cli-args.ts | 80 +++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index e79595ba..8a9202fd 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -29,6 +29,7 @@ import { readFlag, readRepeatedFlag, readSuiteFilters, + validateCliArgs, } from '../lib/cli-args.js'; import { bootPlatformBackend } from './platform-backend.js'; import { viteBuild, vitestRun } from './project-runner.js'; @@ -67,6 +68,35 @@ const HOSTED_PROJECT_REF = 'evalshostedprojectxy'; const HOSTED_ACCESS_TOKEN = 'sbp_' + '0'.repeat(40); const rawArgs = process.argv.slice(2); +const CLI_ARGS = { + booleanFlags: [ + 'skip-existing', + 'smoke', + 'dry', + 'strict', + 'run-all-attempts', + 'debug', + ], + valueFlags: [ + 'mcp', + 'experiment', + 'eval', + 'suite', + 'experiment-suite', + 'runs', + 'timeout-sec', + 'concurrency', + ], + positionals: ['list'], + usage: + 'Usage: pnpm eval -- [list] [--skip-existing] [--smoke] [--dry] [--strict] [--run-all-attempts] [--debug] [--mcp PATH] [--experiment NAME] [--eval ID] [--suite SUITE] [--experiment-suite SUITE] [--runs N] [--timeout-sec N] [--concurrency N]', +} as const; +try { + validateCliArgs(rawArgs, CLI_ARGS); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +} const args = new Set(rawArgs); const FORCE = !args.has('--skip-existing'); const SMOKE = args.has('--smoke'); diff --git a/apps/framework/lib/cli-args.test.ts b/apps/framework/lib/cli-args.test.ts index 8b2f3a1a..95fa444e 100644 --- a/apps/framework/lib/cli-args.test.ts +++ b/apps/framework/lib/cli-args.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { positiveInteger, readFlag } from './cli-args.js'; +import { positiveInteger, readFlag, validateCliArgs } from './cli-args.js'; describe('readFlag', () => { it('reads flags in both --name value and --name=value form', () => { @@ -18,6 +18,39 @@ describe('readFlag', () => { }); }); +describe('validateCliArgs', () => { + const definition = { + booleanFlags: ['strict', 'smoke'], + valueFlags: ['mcp', 'eval'], + positionals: ['list'], + usage: 'Usage: pnpm eval -- [list] [options]', + }; + + it('accepts declared flags, values, separators, and positionals', () => { + expect(() => + validateCliArgs( + ['--', 'list', '--strict', '--mcp', './server', '--eval=id'], + definition + ) + ).not.toThrow(); + }); + + it('rejects unknown flags with a close-match hint and usage', () => { + expect(() => validateCliArgs(['--strcit'], definition)).toThrow( + 'unknown argument: --strcit\nDid you mean --strict?\n\nUsage:' + ); + expect(() => validateCliArgs(['--mpc', './server'], definition)).toThrow( + 'unknown argument: --mpc\nDid you mean --mcp?\n\nUsage:' + ); + }); + + it('rejects unexpected positionals', () => { + expect(() => validateCliArgs(['run'], definition)).toThrow( + 'unexpected argument: run\n\nUsage:' + ); + }); +}); + describe('positiveInteger', () => { it('rejects non-positive-integer CLI options', () => { expect(positiveInteger('3', 'runs')).toBe(3); diff --git a/apps/framework/lib/cli-args.ts b/apps/framework/lib/cli-args.ts index 77eaf1e5..5629e33b 100644 --- a/apps/framework/lib/cli-args.ts +++ b/apps/framework/lib/cli-args.ts @@ -15,6 +15,86 @@ export function positiveInteger(value: string, name: string): number { return parsed.data; } +export interface CliArgsDefinition { + booleanFlags: readonly string[]; + valueFlags: readonly string[]; + positionals?: readonly string[]; + usage: string; +} + +function editDistance(left: string, right: string): number { + const previous = Array.from( + { length: right.length + 1 }, + (_, index) => index + ); + + for (let leftIndex = 0; leftIndex < left.length; leftIndex += 1) { + let diagonal = previous[0] ?? 0; + previous[0] = leftIndex + 1; + for (let rightIndex = 0; rightIndex < right.length; rightIndex += 1) { + const above = previous[rightIndex + 1] ?? 0; + const next = + left[leftIndex] === right[rightIndex] + ? diagonal + : 1 + Math.min(diagonal, above, previous[rightIndex] ?? 0); + diagonal = above; + previous[rightIndex + 1] = next; + } + } + + return previous[right.length] ?? left.length; +} + +function suggestion( + token: string, + flags: readonly string[] +): string | undefined { + const closest = flags + .map((flag) => ({ + flag: `--${flag}`, + distance: editDistance(token, `--${flag}`), + })) + .sort((left, right) => left.distance - right.distance)[0]; + if (!closest || closest.distance > 3) return undefined; + return closest.flag; +} + +/** Rejects tokens that are not part of a command's declared CLI surface. */ +export function validateCliArgs( + rawArgs: readonly string[], + definition: CliArgsDefinition +): void { + const positionals = new Set(definition.positionals ?? []); + const knownFlags = [...definition.booleanFlags, ...definition.valueFlags]; + + for (let index = 0; index < rawArgs.length; index += 1) { + const token = rawArgs[index]; + if (!token || token === '--') continue; + + if (!token.startsWith('--')) { + if (positionals.delete(token)) continue; + throw new Error(`unexpected argument: ${token}\n\n${definition.usage}`); + } + + const equalsIndex = token.indexOf('='); + const name = token.slice(2, equalsIndex === -1 ? undefined : equalsIndex); + if (definition.booleanFlags.includes(name) && equalsIndex === -1) continue; + if (definition.valueFlags.includes(name)) { + const value = rawArgs[index + 1]; + if (equalsIndex === -1 && value && !value.startsWith('--')) index += 1; + continue; + } + + const hint = suggestion( + token.slice(0, equalsIndex === -1 ? undefined : equalsIndex), + knownFlags + ); + throw new Error( + `unknown argument: ${token}${hint ? `\nDid you mean ${hint}?` : ''}\n\n${definition.usage}` + ); + } +} + /** Reads one CLI flag in either `--name value` or `--name=value` form. */ export function readFlag(rawArgs: string[], name: string): string | undefined { const prefix = `--${name}=`; From ab0cf8b409a2366f56647b09b1bb9d2c24c15e06 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Tue, 18 Aug 2026 14:06:23 +0200 Subject: [PATCH 12/12] test(eval): cover CLI process failures --- apps/framework/lib/cli-args.test.ts | 40 +++++++++++++++++++++++++++++ apps/framework/lib/cli-args.ts | 7 +++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/apps/framework/lib/cli-args.test.ts b/apps/framework/lib/cli-args.test.ts index 95fa444e..f1340829 100644 --- a/apps/framework/lib/cli-args.test.ts +++ b/apps/framework/lib/cli-args.test.ts @@ -1,3 +1,5 @@ +import { spawnSync } from 'node:child_process'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { positiveInteger, readFlag, validateCliArgs } from './cli-args.js'; @@ -49,6 +51,44 @@ describe('validateCliArgs', () => { 'unexpected argument: run\n\nUsage:' ); }); + + it('only accepts positionals in the command position', () => { + expect(() => validateCliArgs(['--strict', 'list'], definition)).toThrow( + 'unexpected argument: list\n\nUsage:' + ); + }); +}); + +describe('run-eval argument validation', () => { + const frameworkRoot = join(import.meta.dirname, '..'); + + function run(...args: string[]) { + return spawnSync( + process.execPath, + ['--import', 'tsx/esm', 'harness/run-eval.ts', ...args], + { + cwd: frameworkRoot, + encoding: 'utf8', + } + ); + } + + it.each([ + ['--strcit', '--strict'], + ['--mpc', '--mcp'], + ])('rejects unknown argument %s before running', (token, hint) => { + const result = run(token, './server'); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`unknown argument: ${token}`); + expect(result.stderr).toContain(`Did you mean ${hint}?`); + expect(result.stderr).toContain('Usage: pnpm eval'); + }); + + it('accepts a valid list invocation', () => { + const result = run('list', '--experiment-suite', 'benchmark'); + expect(result.status).toBe(0); + expect(result.stdout).toContain('codex-gpt-5.6'); + }); }); describe('positiveInteger', () => { diff --git a/apps/framework/lib/cli-args.ts b/apps/framework/lib/cli-args.ts index 5629e33b..3a155af5 100644 --- a/apps/framework/lib/cli-args.ts +++ b/apps/framework/lib/cli-args.ts @@ -64,15 +64,18 @@ export function validateCliArgs( rawArgs: readonly string[], definition: CliArgsDefinition ): void { - const positionals = new Set(definition.positionals ?? []); + const positionals = definition.positionals ?? []; const knownFlags = [...definition.booleanFlags, ...definition.valueFlags]; + let hasArgument = false; for (let index = 0; index < rawArgs.length; index += 1) { const token = rawArgs[index]; if (!token || token === '--') continue; + const isFirstArgument = !hasArgument; + hasArgument = true; if (!token.startsWith('--')) { - if (positionals.delete(token)) continue; + if (isFirstArgument && positionals.includes(token)) continue; throw new Error(`unexpected argument: ${token}\n\n${definition.usage}`); }