fix(judge): {"match": "false"} is scored as a PASS — parse verdicts as tri-state (closes #295)

src/clawbench/runner/judge.py

@@ -172,6 +172,25 @@ def _call_anthropic_messages(
     return resp["content"][0]["text"]
 
 
+def _coerce_match(value: object) -> bool | None:
+    """Normalize a judge's `match` field into a tri-state verdict.
+
+    Models answer with real booleans, but also with the *strings* "true" and
+    "false", and occasionally with nothing at all. `bool("false")` is True, so
+    a stringly-typed mismatch used to score as a pass; a missing key used to
+    score as a hard mismatch rather than as inconclusive.
+    """
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        token = value.strip().strip('".').lower()
+        if token in {"true", "yes", "pass", "match"}:
+            return True
+        if token in {"false", "no", "fail", "mismatch"}:
+            return False
+    return None
+
+
 def _parse_verdict(text: str) -> tuple[bool | None, str]:
     """Best-effort parse of the judge's reply into (match, reason)."""
     text = (text or "").strip()
@@ -184,7 +203,7 @@ def _parse_verdict(text: str) -> tuple[bool | None, str]:
         start = text.index("{")
         end = text.rindex("}") + 1
         obj = json.loads(text[start:end])
-        return bool(obj.get("match")), str(obj.get("reason", ""))
+        return _coerce_match(obj.get("match")), str(obj.get("reason", ""))
     except (ValueError, json.JSONDecodeError):
         # Heuristic fallback
         low = text.lower()

src/clawbench/runner/judge_llm.py

@@ -30,6 +30,8 @@
 import urllib.error
 from typing import Any
 
+from clawbench.runner.judge import _coerce_match
+
 
 JUDGE_SYSTEM = """You are a lenient evaluator for a web-agent benchmark.
 
@@ -143,8 +145,12 @@ def _call_anthropic_messages(
     )
 
 
-def _parse_verdict(raw: str) -> tuple[bool, str]:
-    """Best-effort parse of the judge's reply into (match, reason). Default TRUE on parse failure."""
+def _parse_verdict(raw: str) -> tuple[bool | None, str]:
+    """Best-effort parse of the judge's reply into (match, reason).
+
+    Returns None when the reply carries no usable verdict, so an unparseable
+    judge response is reported as inconclusive instead of silently passing.
+    """
     try:
         # Strip markdown fences if any
         s = raw.strip()
@@ -153,17 +159,19 @@ def _parse_verdict(raw: str) -> tuple[bool, str]:
             if s.endswith("```"):
                 s = s.rsplit("\n", 1)[0] if "\n" in s else s.rstrip("`")
         obj = json.loads(s)
-        return bool(obj.get("match", True)), str(obj.get("reason", ""))
+        return _coerce_match(obj.get("match")), str(obj.get("reason", ""))
     except Exception:
-        # Fall back to keyword scan, defaulting to TRUE (per lenient rubric)
+        # Keyword fallback for replies that are not valid JSON. An unparseable
+        # reply is inconclusive (None), never an implicit pass.
         low = raw.lower()
-        if (
-            "false" in low
-            and "match" in low
-            and low.find("false") - low.find("match") < 80
-        ):
+        if "match" not in low:
+            return None, raw[:200] or "unparseable"
+        after = low.split("match", 1)[1][:80]
+        if "false" in after:
             return False, raw[:200]
-        return True, raw[:200]
+        if "true" in after:
+            return True, raw[:200]
+        return None, raw[:200] or "unparseable"
 
 
 def judge_request(

tests/test_judge.py

@@ -2,7 +2,9 @@
 
 from __future__ import annotations
 
-from clawbench.runner import judge
+import pytest
+
+from clawbench.runner import judge, judge_llm
 
 
 def test_gemini_openai_cfg_normalizes_native_root() -> None:
@@ -52,3 +54,57 @@ def test_unsupported_api_type_reports_error(monkeypatch) -> None:
         {"request": {"url": "x"}},
     )
     assert r["match"] is None and r["error"] == "unsupported_api_type"
+
+
+# --- verdict parsing (see #295) -------------------------------------------
+#
+# The `match` field decides whether a run counts as a pass, so it has to be
+# read exactly. Models answer with real booleans, with the *strings* "true"
+# and "false", and sometimes with no verdict at all — `bool("false")` is True,
+# so a stringly-typed mismatch used to be scored as a pass.
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        # real booleans
+        ('{"match": true, "reason": "ok"}', True),
+        ('{"match": false, "reason": "wrong item"}', False),
+        # stringly-typed verdicts — the regression this pins
+        ('{"match": "false", "reason": "wrong item"}', False),
+        ('{"match": "true"}', True),
+        ('{"match": "FALSE"}', False),
+        ('{"match": "False"}', False),
+        # no usable verdict → inconclusive, never an implicit pass/fail
+        ('{"reason": "forgot the verdict"}', None),
+        ('{"match": null}', None),
+        ('{"match": "maybe"}', None),
+        # fenced JSON still parses
+        ('```json\n{"match": false, "reason": "r"}\n```', False),
+    ],
+)
+def test_parse_verdict_is_tri_state(raw: str, expected: bool | None) -> None:
+    assert judge._parse_verdict(raw)[0] is expected
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        ('{"match": "false", "reason": "wrong item"}', False),
+        ('{"match": false}', False),
+        ('{"match": true}', True),
+        # a reply with no verdict key must not default to a pass: this module
+        # produces the published Reward-lenient column
+        ('{"reason": "forgot the verdict"}', None),
+        ("not json at all", None),
+    ],
+)
+def test_lenient_judge_parse_verdict_is_tri_state(
+    raw: str, expected: bool | None
+) -> None:
+    assert judge_llm._parse_verdict(raw)[0] is expected
+
+
+def test_coerce_match_rejects_non_verdict_types() -> None:
+    for value in (1, 0, [], {}, None, "  "):
+        assert judge._coerce_match(value) is None