From 77419def3889ec5aa1cec8bc9ea3f7c76e75f188 Mon Sep 17 00:00:00 2001 From: iroiro147 Date: Sun, 2 Aug 2026 11:10:15 +0530 Subject: [PATCH] fix(evals): coalesce null output for Braintrust reporter and guard log() throws MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A no-turn eval (e.g. `t.target.dispatchSchedule(...)` + `t.check` DB assertions) legitimately produces `result.output === null` per the eval API's own derivation. Braintrust's `validateAndSanitizeExperimentLogFullArgs` rejects null/undefined output ("output must be specified"), and the throw escaped `onEvalComplete`, killing the entire `eve eval` run — remaining evals never executed and no artifacts were written, even though the crashing eval itself passed all its gates. Two fixes: 1. `output: result.result.output ?? ""` — coalesce null to an empty string so the SDK accepts it. 2. Wrap `experiment.log()` in try/catch — any reporter throw is logged to stderr but does not abort the run. The remaining evals still execute and artifacts are written. Regression tests: - `coalesces null output to empty string for no-turn evals` — drives an eval result with `output: null`, asserts `log` receives `output: ""` - `survives a log() throw without aborting the run` — mocks `log` to throw, asserts no rethrow, then asserts a second call still reaches `log` (the next eval is not dropped) Verified: - `pnpm exec vitest run --config vitest.unit.config.ts src/evals/runner/reporters/braintrust.test.ts` — 7/7 tests green - `pnpm exec tsc -p tsconfig.json --noEmit` — clean Refs #1405 Signed-off-by: Sarthak Singh --- .changeset/strong-owls-hug.md | 11 ++++ .../evals/runner/reporters/braintrust.test.ts | 56 +++++++++++++++++++ .../src/evals/runner/reporters/braintrust.ts | 31 ++++++---- 3 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 .changeset/strong-owls-hug.md diff --git a/.changeset/strong-owls-hug.md b/.changeset/strong-owls-hug.md new file mode 100644 index 000000000..dc14d0d6f --- /dev/null +++ b/.changeset/strong-owls-hug.md @@ -0,0 +1,11 @@ +--- +"eve": patch +--- + +fix(evals): coalesce null output for Braintrust reporter and guard against log() throws (#1405) + +A no-turn eval (e.g. schedule-dispatch + DB assertions) legitimately produces `result.output === null` per the eval API's own derivation. The Braintrust SDK rejects null/undefined output (`"output must be specified"`), and the throw escaped `onEvalComplete`, killing the entire `eve eval` run — remaining evals never executed and no artifacts were written. + +Two fixes: +1. Coalesce `output: result.result.output ?? ""` so null is sent as an empty string. +2. Wrap `experiment.log()` in try/catch so any reporter throw is logged but does not abort the run. diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.ts index a70421659..a2c27e22c 100644 --- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts +++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts @@ -199,4 +199,60 @@ describe("Braintrust", () => { }), ); }); + + it("coalesces null output to empty string for no-turn evals (#1405)", async () => { + const reporter = Braintrust(makeConfig()); + await reporter.onRunStart([makeEval()], makeTarget()); + + reporter.onEvalComplete( + makeEvalResult({ + result: { + // A no-turn eval (schedule-dispatch + DB assertions) produces + // output === null per the eval API's own derivation. + output: null, + finalMessage: null, + status: "completed", + events: [], + derived: { + toolCalls: [], + toolCallCount: 0, + subagentCalls: [], + subagentCallCount: 0, + inputRequests: [], + parked: false, + messageCount: 0, + reasoningBlockCount: 0, + }, + sessionId: "session-456", + }, + verdict: "passed", + }), + ); + + // Braintrust's SDK rejects null output ("output must be specified"). + // The reporter coalesces to "" so the run doesn't crash. + expect(braintrustMocks.log).toHaveBeenCalledWith( + expect.objectContaining({ + output: "", + }), + ); + }); + + it("survives a log() throw without aborting the run (#1405)", async () => { + const reporter = Braintrust(makeConfig()); + await reporter.onRunStart([makeEval()], makeTarget()); + + // Simulate Braintrust SDK throwing on log — the reporter must catch + // so the remaining evals still execute. + braintrustMocks.log.mockImplementationOnce(() => { + throw new Error("output must be specified"); + }); + + // Should NOT throw + expect(() => reporter.onEvalComplete(makeEvalResult())).not.toThrow(); + + // A second call (the next eval) should still reach log + reporter.onEvalComplete(makeEvalResult()); + expect(braintrustMocks.log).toHaveBeenCalledTimes(2); + }); }); diff --git a/packages/eve/src/evals/runner/reporters/braintrust.ts b/packages/eve/src/evals/runner/reporters/braintrust.ts index 2b611cde6..2375e3b43 100644 --- a/packages/eve/src/evals/runner/reporters/braintrust.ts +++ b/packages/eve/src/evals/runner/reporters/braintrust.ts @@ -172,16 +172,27 @@ class BraintrustReporter implements EvalReporter { reasoningBlockCount: result.result.derived.reasoningBlockCount, }; - this.#experiment.log({ - id: result.id, - input: evaluation?.description ?? "", - output: result.result.output, - error: result.error ?? undefined, - scores, - metadata, - metrics, - tags: evaluation?.tags ? [...evaluation.tags] : undefined, - }); + try { + this.#experiment.log({ + id: result.id, + input: evaluation?.description ?? "", + // Braintrust's SDK rejects null/undefined output ("output must be + // specified"). A no-turn eval (e.g. schedule-dispatch + DB assertions) + // legitimately produces output === null per the eval API's own + // derivation. Coalesce to an empty string so the reporter doesn't + // crash the entire eval run. See #1405. + output: result.result.output ?? "", + error: result.error ?? undefined, + scores, + metadata, + metrics, + tags: evaluation?.tags ? [...evaluation.tags] : undefined, + }); + } catch (error) { + // A reporter throw in onEvalComplete must not abort the run and + // drop the remaining evals + artifacts. Log and continue. See #1405. + console.error(`Braintrust reporter: failed to log eval "${result.id}":`, error); + } } async onRunComplete(_summary: EveEvalRunSummary): Promise {