fix(eval): one Stage-1 matching predicate for the live and offline paths

src/clawbench/eval/edgebench_judge.py

@@ -23,13 +23,37 @@
 import hmac
 import json
 import os
-import re
 import sys
 from pathlib import Path
 from typing import Any
-from urllib.parse import parse_qs, urlparse
 
 from clawbench.runner.judge import judge_request
+from clawbench.utils.paths import RUNTIME_ROOT
+
+
+def _load_runtime_matching():
+    """Load the Stage-1 predicate from the runtime-server directory.
+
+    It lives beside the interceptor that runs it, because runtime-server/ is
+    what gets COPYed into every task image. That directory name is not a valid
+    module path, so it is loaded by file. Sharing the one copy is the point: a
+    re-implementation here is what drifted from the live interceptor and made
+    offline verdicts disagree with real runs.
+    """
+    import importlib.util
+
+    path = RUNTIME_ROOT / "runtime-server" / "matching.py"
+    spec = importlib.util.spec_from_file_location(
+        "clawbench_runtime_matching", str(path)
+    )
+    if spec is None or spec.loader is None:  # pragma: no cover - packaging error
+        raise ImportError(f"cannot load the Stage-1 matcher from {path}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+_matching = _load_runtime_matching()
 
 
 def _verify_signature(intercept: dict[str, Any], secret: str) -> bool:
@@ -52,46 +76,15 @@ def _verify_signature(intercept: dict[str, Any], secret: str) -> bool:
     return hmac.compare_digest(sig, expected)
 
 
-def _const_fields_match(expected: Any, actual: Any) -> bool:
-    """All key/values in ``expected`` present in ``actual`` (mirrors runtime-server)."""
-    if not expected:
-        return True
-    if not actual:
-        return False
-    if isinstance(actual, list):
-        return any(_const_fields_match(expected, item) for item in actual)
-    if not isinstance(actual, dict):
-        return False
-    return all(actual.get(k) == v for k, v in expected.items())
-
-
 def _stage1_match(request: dict[str, Any], eval_schema: Any) -> bool:
     """Recompute Stage-1 against the task schema — do NOT trust the agent's flag.
 
     The agent controls the submitted evidence archive, so re-verify that the
-    submitted request actually hits the task's target (url_pattern regex + method
-    + const body/params), exactly as the runtime interceptor would.
+    submitted request actually hits the task's target, using the very predicate
+    the in-container interceptor ran. This was a hand-maintained mirror of
+    runtime-server until the two drifted; see ``_matching`` above.
     """
-    if not isinstance(eval_schema, dict):
-        return False
-    url_pattern = eval_schema.get("url_pattern") or ""
-    if not url_pattern:
-        return False  # no target to verify against → cannot confirm interception
-    url = str(request.get("url") or "")
-    try:
-        if not re.search(url_pattern, url):
-            return False
-    except re.error:
-        return False
-    method = eval_schema.get("method")
-    if method and request.get("method") != method:
-        return False
-    if not _const_fields_match(eval_schema.get("body"), request.get("body")):
-        return False
-    # Always derive query params from the URL (like the runtime interceptor) — do
-    # not trust a submitted request["params"] field, which could be forged.
-    params = {k: v[0] for k, v in parse_qs(urlparse(url).query).items()}
-    return _const_fields_match(eval_schema.get("params"), params)
+    return _matching.stage1_match(request, eval_schema)
 
 
 # SForge structured_json markers (grading._grade_structured looks for these).

src/clawbench/runtime/harbor/Dockerfile

@@ -27,6 +27,9 @@ RUN UV_PYTHON_PREFERENCE=only-system uv sync --frozen \
 
 WORKDIR /app
 COPY runtime-server/server.py ./src/runtime-server/server.py
+# Stage-1 matching predicate, imported by server.py and shared with the
+# offline verifier. Must ship alongside server.py or interception breaks.
+COPY runtime-server/matching.py ./src/runtime-server/matching.py
 COPY chrome-extension/ ./src/chrome-extension/
 COPY shared/ ./src/shared/
 COPY harbor/ ./src/harbor/

src/clawbench/runtime/harnesses/base/Dockerfile.base

@@ -29,6 +29,9 @@ RUN UV_PYTHON_PREFERENCE=only-system uv sync --frozen
 
 WORKDIR /app
 COPY runtime-server/server.py ./src/runtime-server/server.py
+# Stage-1 matching predicate, imported by server.py and shared with the
+# offline verifier. Must ship alongside server.py or interception breaks.
+COPY runtime-server/matching.py ./src/runtime-server/matching.py
 
 COPY chrome-extension/ ./src/chrome-extension/
 

src/clawbench/runtime/runtime-server/matching.py

@@ -0,0 +1,95 @@
+"""Stage-1 interceptor matching — the benchmark's deterministic ground truth.
+
+Stage 1 asks one question: does this HTTP request hit the task's target
+(``url_pattern`` regex + ``method`` + constant ``body``/``params`` fields)?
+Every published Intercepted number is that answer.
+
+It is computed in two places — live, in-container, by ``server.py`` next to
+this file, and offline by ``clawbench.eval.edgebench_judge`` when it
+re-verifies a submitted evidence archive. Those were hand-maintained copies
+and they drifted, so offline judging could disagree with what actually
+happened during a run. This module is the single copy both import.
+
+Kept to the standard library on purpose: it is imported by the offline
+verifier on the host, where the runtime-server's dependencies are absent.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+
+def const_fields_match(expected: Any, actual: Any) -> bool:
+    """All key/value pairs in ``expected`` are present in ``actual``.
+
+    For list bodies (batched GraphQL) any single item matching is enough.
+    An empty or absent ``expected`` constrains nothing and matches.
+    """
+    if not expected:
+        return True
+    if not actual:
+        return False
+    if isinstance(actual, list):
+        return any(const_fields_match(expected, item) for item in actual)
+    if not isinstance(actual, dict):
+        return False
+    return all(actual.get(k) == v for k, v in expected.items())
+
+
+def query_params_from_url(url: str) -> dict[str, Any]:
+    """Query string as a dict, collapsing single-valued keys to a scalar.
+
+    A repeated key keeps its list of values, so ``?tag=a&tag=b`` is
+    ``{"tag": ["a", "b"]}`` and does not match a constant of ``{"tag": "a"}``.
+    The offline verifier used to take ``v[0]`` unconditionally and so called
+    that request intercepted when the live interceptor had let it through.
+    """
+    parsed = urlparse(str(url or ""))
+    return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(parsed.query).items()}
+
+
+def url_pattern_matches(url_pattern: str, url: str) -> bool:
+    """Whether ``url`` matches the task's target regex.
+
+    A malformed pattern is a task-authoring error, not a request that should
+    be intercepted, so it is reported as no-match. It must never raise: this
+    runs inside the CDP event loop, where an exception stops interception for
+    the remainder of the run and silently zeroes the task's Stage-1 score.
+    """
+    if not url_pattern:
+        return False
+    try:
+        return re.search(url_pattern, str(url or "")) is not None
+    except re.error:
+        return False
+
+
+def stage1_match(request: dict[str, Any], eval_schema: Any) -> bool:
+    """Whether ``request`` hits the target described by ``eval_schema``.
+
+    ``request`` carries ``url``, ``method``, and a parsed ``body``. Query
+    params are always derived from the URL rather than read off the request:
+    offline, a submitted ``params`` field is agent-controlled and could be
+    forged.
+
+    An absent or empty ``url_pattern`` means there is no target to verify
+    against and returns False. Live, that case is handled earlier — the
+    interceptor is simply never armed and no request is ever blocked.
+    """
+    if not isinstance(eval_schema, dict):
+        return False
+
+    url = str(request.get("url") or "")
+    if not url_pattern_matches(eval_schema.get("url_pattern") or "", url):
+        return False
+
+    method = eval_schema.get("method")
+    if method and request.get("method") != method:
+        return False
+
+    if not const_fields_match(eval_schema.get("body"), request.get("body")):
+        return False
+
+    return const_fields_match(eval_schema.get("params"), query_params_from_url(url))

src/clawbench/runtime/runtime-server/server.py

@@ -2,20 +2,23 @@
 import base64
 import json
 import os
-import re
 import signal
 import subprocess
 import threading
 import time
 from contextlib import asynccontextmanager
 from pathlib import Path
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs
 import urllib.request
 
 import websocket
 from fastapi import FastAPI, WebSocket, WebSocketDisconnect
 from fastapi.responses import HTMLResponse, JSONResponse
 
+# Sibling module, shared verbatim with the offline verifier
+# (clawbench.eval.edgebench_judge) so the two cannot drift.
+from matching import query_params_from_url, stage1_match
+
 DATA_DIR = Path(os.environ.get("CLAWBENCH_DATA_DIR", "/data"))
 ACTIONS_FILE = DATA_DIR / "actions.jsonl"
 SCREENSHOTS_DIR = DATA_DIR / "screenshots"
@@ -170,21 +173,6 @@ def stop_ffmpeg_recording(timeout: int = 10) -> str:
 """
 
 
-def _const_fields_match(expected, actual):
-    """Check that all key-value pairs in expected match in actual data.
-    For list bodies (batched GraphQL), returns True if any item matches.
-    Returns True if all match or expected is empty/None."""
-    if not expected:
-        return True
-    if not actual:
-        return False
-    if isinstance(actual, list):
-        return any(_const_fields_match(expected, item) for item in actual)
-    if not isinstance(actual, dict):
-        return False
-    return all(actual.get(k) == v for k, v in expected.items())
-
-
 FILTERED_PREFIXES = (
     "http://localhost:7878",
     "http://127.0.0.1:7878",
@@ -218,18 +206,13 @@ def _log_request(log_file, params):
     if any(request_url.startswith(p) for p in FILTERED_PREFIXES):
         return
 
-    parsed = urlparse(request_url)
-    query_params = {
-        k: v[0] if len(v) == 1 else v for k, v in parse_qs(parsed.query).items()
-    }
-
     entry = {
         "timestamp": time.time(),
         "url": request_url,
         "method": request["method"],
         "headers": request.get("headers", {}),
         "body": _parse_body(request.get("postData")),
-        "query_params": query_params,
+        "query_params": query_params_from_url(request_url),
         "resource_type": params.get("resourceType", "Other"),
     }
     log_file.write(json.dumps(entry) + "\n")
@@ -451,34 +434,31 @@ def activate_session_target(session_id, reason):
                 continue
 
             # --- Intercept: block if URL + method + body/params match ---
-            if not re.search(url_pattern, request_url):
-                send("Fetch.continueRequest", {"requestId": request_id}, session_id)
-                continue
-
-            if required_method and params["request"]["method"] != required_method:
-                send("Fetch.continueRequest", {"requestId": request_id}, session_id)
-                continue
-
-            # Parse request data for body/params matching
-            parsed = urlparse(request_url)
-            query_params = {
-                k: v[0] if len(v) == 1 else v for k, v in parse_qs(parsed.query).items()
-            }
+            # One shared predicate, not four inline branches: every failing
+            # check here continues the request, and the offline verifier has
+            # to reach the identical verdict from the archived evidence.
             body = _parse_body(params["request"].get("postData"))
-
-            if not _const_fields_match(match_body, body):
-                send("Fetch.continueRequest", {"requestId": request_id}, session_id)
-                continue
-
-            if not _const_fields_match(match_params, query_params):
+            if not stage1_match(
+                {
+                    "url": request_url,
+                    "method": params["request"]["method"],
+                    "body": body,
+                },
+                {
+                    "url_pattern": url_pattern,
+                    "method": required_method,
+                    "body": match_body,
+                    "params": match_params,
+                },
+            ):
                 send("Fetch.continueRequest", {"requestId": request_id}, session_id)
                 continue
 
             # All filters matched — block the request
             request_obj = {
                 "url": request_url,
                 "method": params["request"]["method"],
-                "params": query_params,
+                "params": query_params_from_url(request_url),
                 "body": body,
             }