diff --git a/src/clawbench/runner/judge.py b/src/clawbench/runner/judge.py index 0ba0e4f2..6b15453a 100644 --- a/src/clawbench/runner/judge.py +++ b/src/clawbench/runner/judge.py @@ -172,6 +172,25 @@ def _call_anthropic_messages( return resp["content"][0]["text"] +def _coerce_match(value: object) -> bool | None: + """Normalize a judge's `match` field into a tri-state verdict. + + Models answer with real booleans, but also with the *strings* "true" and + "false", and occasionally with nothing at all. `bool("false")` is True, so + a stringly-typed mismatch used to score as a pass; a missing key used to + score as a hard mismatch rather than as inconclusive. + """ + if isinstance(value, bool): + return value + if isinstance(value, str): + token = value.strip().strip('".').lower() + if token in {"true", "yes", "pass", "match"}: + return True + if token in {"false", "no", "fail", "mismatch"}: + return False + return None + + def _parse_verdict(text: str) -> tuple[bool | None, str]: """Best-effort parse of the judge's reply into (match, reason).""" text = (text or "").strip() @@ -184,7 +203,7 @@ def _parse_verdict(text: str) -> tuple[bool | None, str]: start = text.index("{") end = text.rindex("}") + 1 obj = json.loads(text[start:end]) - return bool(obj.get("match")), str(obj.get("reason", "")) + return _coerce_match(obj.get("match")), str(obj.get("reason", "")) except (ValueError, json.JSONDecodeError): # Heuristic fallback low = text.lower() diff --git a/src/clawbench/runner/judge_llm.py b/src/clawbench/runner/judge_llm.py index 59573329..b5626b17 100644 --- a/src/clawbench/runner/judge_llm.py +++ b/src/clawbench/runner/judge_llm.py @@ -30,6 +30,8 @@ import urllib.error from typing import Any +from clawbench.runner.judge import _coerce_match + JUDGE_SYSTEM = """You are a lenient evaluator for a web-agent benchmark. @@ -143,8 +145,12 @@ def _call_anthropic_messages( ) -def _parse_verdict(raw: str) -> tuple[bool, str]: - """Best-effort parse of the judge's reply into (match, reason). Default TRUE on parse failure.""" +def _parse_verdict(raw: str) -> tuple[bool | None, str]: + """Best-effort parse of the judge's reply into (match, reason). + + Returns None when the reply carries no usable verdict, so an unparseable + judge response is reported as inconclusive instead of silently passing. + """ try: # Strip markdown fences if any s = raw.strip() @@ -153,17 +159,19 @@ def _parse_verdict(raw: str) -> tuple[bool, str]: if s.endswith("```"): s = s.rsplit("\n", 1)[0] if "\n" in s else s.rstrip("`") obj = json.loads(s) - return bool(obj.get("match", True)), str(obj.get("reason", "")) + return _coerce_match(obj.get("match")), str(obj.get("reason", "")) except Exception: - # Fall back to keyword scan, defaulting to TRUE (per lenient rubric) + # Keyword fallback for replies that are not valid JSON. An unparseable + # reply is inconclusive (None), never an implicit pass. low = raw.lower() - if ( - "false" in low - and "match" in low - and low.find("false") - low.find("match") < 80 - ): + if "match" not in low: + return None, raw[:200] or "unparseable" + after = low.split("match", 1)[1][:80] + if "false" in after: return False, raw[:200] - return True, raw[:200] + if "true" in after: + return True, raw[:200] + return None, raw[:200] or "unparseable" def judge_request( diff --git a/tests/test_judge.py b/tests/test_judge.py index 450ed504..6fd767e4 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -2,7 +2,9 @@ from __future__ import annotations -from clawbench.runner import judge +import pytest + +from clawbench.runner import judge, judge_llm def test_gemini_openai_cfg_normalizes_native_root() -> None: @@ -52,3 +54,57 @@ def test_unsupported_api_type_reports_error(monkeypatch) -> None: {"request": {"url": "x"}}, ) assert r["match"] is None and r["error"] == "unsupported_api_type" + + +# --- verdict parsing (see #295) ------------------------------------------- +# +# The `match` field decides whether a run counts as a pass, so it has to be +# read exactly. Models answer with real booleans, with the *strings* "true" +# and "false", and sometimes with no verdict at all — `bool("false")` is True, +# so a stringly-typed mismatch used to be scored as a pass. + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + # real booleans + ('{"match": true, "reason": "ok"}', True), + ('{"match": false, "reason": "wrong item"}', False), + # stringly-typed verdicts — the regression this pins + ('{"match": "false", "reason": "wrong item"}', False), + ('{"match": "true"}', True), + ('{"match": "FALSE"}', False), + ('{"match": "False"}', False), + # no usable verdict → inconclusive, never an implicit pass/fail + ('{"reason": "forgot the verdict"}', None), + ('{"match": null}', None), + ('{"match": "maybe"}', None), + # fenced JSON still parses + ('```json\n{"match": false, "reason": "r"}\n```', False), + ], +) +def test_parse_verdict_is_tri_state(raw: str, expected: bool | None) -> None: + assert judge._parse_verdict(raw)[0] is expected + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ('{"match": "false", "reason": "wrong item"}', False), + ('{"match": false}', False), + ('{"match": true}', True), + # a reply with no verdict key must not default to a pass: this module + # produces the published Reward-lenient column + ('{"reason": "forgot the verdict"}', None), + ("not json at all", None), + ], +) +def test_lenient_judge_parse_verdict_is_tri_state( + raw: str, expected: bool | None +) -> None: + assert judge_llm._parse_verdict(raw)[0] is expected + + +def test_coerce_match_rejects_non_verdict_types() -> None: + for value in (1, 0, [], {}, None, " "): + assert judge._coerce_match(value) is None