From 84c2218d3d9d79939523413e5415cb5a71930192 Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 19 Aug 2026 13:55:19 +0200 Subject: [PATCH 1/3] ci: bump Go to 1.26.6 to clear the govulncheck gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/govulncheck.sh has an empty ALLOWLIST and fails on any reachable advisory, so it goes red whenever new stdlib advisories are published — no code change required. Seven landed against go1.26.5 and have blocked every PR since master last went green on 2026-08-06: GO-2026-5026 net/http GO-2026-6089 net/http GO-2026-5972 encoding/asn1 GO-2026-6090 crypto/tls GO-2026-6088 encoding/xml GO-2026-6091 html/template GO-2026-6218 net/url All seven are standard-library and all are fixed in go1.26.6. Every workflow resolves its toolchain from `go-version-file: go.mod`, so the go directive is the single place to bump. Dockerfile pinned golang:1.26-bookworm by digest at go1.26.4, so release binaries would keep the vulnerable stdlib even after the go.mod bump; repin to the go1.26.6 digest. Dockerfile.mcp and Dockerfile.proxy float on golang:1.26-* and pick it up on their own. Verified: `scripts/govulncheck.sh` reports "no reachable vulnerabilities", and `make build` + `make test` are green on 1.26.6. --- Dockerfile | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 565178a5..8a9818be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ # ============================================================================= # Stage 1: Go builder # ============================================================================= -FROM golang:1.26-bookworm@sha256:5d2b868674b57c9e48cdd39e891acce4196b6926ca6d11e9c270a8f85106203d AS builder +FROM golang:1.26-bookworm@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36 AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ git ca-certificates && \ diff --git a/go.mod b/go.mod index c40317c7..d14c6ac0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ethpandaops/panda -go 1.26.5 +go 1.26.6 require ( github.com/containerd/errdefs v1.0.0 From ca876ca86e10cd3ad1a641bdde0da185f360d191 Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 19 Aug 2026 13:55:30 +0200 Subject: [PATCH 2/3] eval: fail a turn that never reached the model, instead of grading it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every smoke case in CI comes back with 0 tokens, 0 tool calls and an empty answer, and the harness reports `crashed=False` / "0 errors" — so an infrastructure failure is graded as eight wrong answers. That is wrong twice over: it reads as an eval regression when nothing about the eval changed, and `crashed=False` skips promptfoo/provider.py's retry-on-crash path, which exists precisely for transient provider faults. execute() only ever read `text` and `tool` parts off the returned messages, so a provider/model failure — which opencode reports as an `error` on the message itself — was dropped on the floor and left an empty ExecutionResult that looked like a legitimately empty answer. Now: capture that error, and treat a turn with no text, no tool call and no tokens as an execution error. Both set is_error, so the run is marked crashed, retried once, and reported with the provider's own message. This does not fix the outage behind the current red build. The subject model pin (opencode-go/deepseek-v4-flash) has not changed since 2026-07-15, and the grader rides the same gateway and key successfully, so the model appears to have gone away provider-side between the 2026-07-21 green run and the 2026-08-06 red one. This change is what makes the next run say so. --- tests/eval/agent/opencode_agent.py | 51 ++++++++ tests/eval/tests/test_agent_no_output.py | 141 +++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 tests/eval/tests/test_agent_no_output.py diff --git a/tests/eval/agent/opencode_agent.py b/tests/eval/agent/opencode_agent.py index 520aa138..c2a4a795 100644 --- a/tests/eval/agent/opencode_agent.py +++ b/tests/eval/agent/opencode_agent.py @@ -94,6 +94,42 @@ def _free_port() -> int: # A single `opencode serve` is shared across all OpenCodeAgent instances with the # same config (keyed by the rendered opencode.json), so a pytest run with a # function-scoped agent fixture pays the server cold-start once, not per test. +def _error_text(error: Any) -> str: + """Render opencode's error payload (shape varies by provider) for a message.""" + if isinstance(error, dict): + for key in ("message", "detail", "error"): + value = error.get(key) + if isinstance(value, str) and value: + return value + return json.dumps(error)[:300] + return str(error)[:300] + + +def _raise_if_no_output( + *, + provider_error: Any, + final_text: str, + tool_calls: list[ToolCallRecord], + tokens: int, + model: str, +) -> None: + """Fail a turn that never reached the model, instead of scoring it as an answer. + + A turn with no text, no tool call and no tokens means the model was never + invoked — a bad model id, an expired key, a provider outage. Left unreported + that grades as a wrong answer, so a broken harness looks like a quality + regression and the provider's retry path never fires. + """ + if provider_error is not None: + raise RuntimeError(f"opencode provider error for {model}: {_error_text(provider_error)}") + + if not final_text and not tool_calls and not tokens: + raise RuntimeError( + f"opencode produced no output for {model}: no text, no tool calls and no " + "tokens, so the model was never invoked (check the model id and API key)" + ) + + _SHARED_SERVERS: dict[str, subprocess.Popen[bytes]] = {} _SHARED_URLS: dict[str, str] = {} _SHARED_CONTAINERS: dict[str, str] = {} # server key -> docker container name (sandbox mode) @@ -491,11 +527,18 @@ async def execute( input_tokens = 0 output_tokens = 0 + provider_error: Any = None + for item in after: d = self._as_dict(item) info = d.get("info", {}) or {} if info.get("id") in seen: continue + # opencode reports a provider/model failure on the message itself. + # Without this the turn is indistinguishable from a legitimately + # empty answer, and a provider outage grades as a wrong answer. + if info.get("error") and provider_error is None: + provider_error = info["error"] if info.get("role") != "assistant": continue cost += float(info.get("cost") or 0.0) @@ -521,6 +564,14 @@ async def execute( elif ty == "text" and p.get("text"): final_text = p["text"] + _raise_if_no_output( + provider_error=provider_error, + final_text=final_text, + tool_calls=tool_calls, + tokens=input_tokens + output_tokens, + model=f"{self.provider_id}/{self.model_id}", + ) + result.session_id = sid result.output = final_text result.tool_calls = tool_calls diff --git a/tests/eval/tests/test_agent_no_output.py b/tests/eval/tests/test_agent_no_output.py new file mode 100644 index 00000000..bb4f5bd9 --- /dev/null +++ b/tests/eval/tests/test_agent_no_output.py @@ -0,0 +1,141 @@ +"""A turn that never reached the model must fail loudly, not score as an answer. + +CI hit this for real: every smoke case came back with 0 tokens, 0 tool calls and +an empty answer, and the harness graded them as wrong answers (`crashed=False`, +"0 errors"). That reads as an eval regression when it is actually an +infrastructure failure, and it also skips the provider's retry-on-crash path. +""" + +from typing import Any + +import pytest + +from agent.opencode_agent import OpenCodeAgent, _error_text, _raise_if_no_output +from agent.wrapper import ToolCallRecord +from config.settings import EvalSettings + + +def test_provider_error_is_raised(): + with pytest.raises(RuntimeError, match="provider error.*model not found"): + _raise_if_no_output( + provider_error={"message": "model not found"}, + final_text="", + tool_calls=[], + tokens=0, + model="opencode-go/deepseek-v4-flash", + ) + + +def test_silent_empty_turn_is_raised(): + with pytest.raises(RuntimeError, match="produced no output"): + _raise_if_no_output( + provider_error=None, final_text="", tool_calls=[], tokens=0, model="p/m" + ) + + +@pytest.mark.parametrize( + "final_text,tool_calls,tokens", + [ + ("an answer", [], 0), # answered, no tools + ("", [ToolCallRecord(name="t", input={})], 0), # tools ran, no summary text + ("", [], 42), # model billed tokens but said nothing + ], +) +def test_turn_with_any_signal_is_accepted(final_text, tool_calls, tokens): + _raise_if_no_output( + provider_error=None, + final_text=final_text, + tool_calls=tool_calls, + tokens=tokens, + model="p/m", + ) + + +@pytest.mark.parametrize( + "payload,expected", + [ + ({"message": "boom"}, "boom"), + ({"detail": "nested"}, "nested"), + ({"code": 502}, '{"code": 502}'), + ("plain", "plain"), + ], +) +def test_error_text_renders_provider_shapes(payload, expected): + assert _error_text(payload) == expected + + +class _FakeSession: + """Mimics an opencode session; the turn's messages only exist after chat(). + + execute() snapshots message ids before the turn and attributes only new ones, + so the pre-chat read must be empty or every message is filtered out as `seen`. + """ + + def __init__(self, messages: list[dict[str, Any]]) -> None: + self._messages = messages + self._chatted = False + + async def create(self): + return type("S", (), {"id": "sess-1"})() + + async def messages(self, id: str): # noqa: A002 - matches the SDK signature + return self._messages if self._chatted else [] + + async def chat(self, **kwargs): + self._chatted = True + return None + + +class _FakeClient: + def __init__(self, messages: list[dict[str, Any]]) -> None: + self.session = _FakeSession(messages) + + +async def _run(monkeypatch, messages: list[dict[str, Any]]): + agent = OpenCodeAgent(EvalSettings()) + + async def _noop_ensure_server() -> None: + return None + + monkeypatch.setattr(agent, "_ensure_server", _noop_ensure_server) + agent._client = _FakeClient(messages) + agent._langfuse = None + return await agent.execute("what datasources are available?") + + +async def test_execute_marks_empty_turn_as_error(monkeypatch): + empty_turn = [{"info": {"id": "m1", "role": "assistant"}, "parts": []}] + + result = await _run(monkeypatch, empty_turn) + + assert result.is_error + assert "produced no output" in (result.error_message or "") + assert result.output == "" + + +async def test_execute_surfaces_provider_error(monkeypatch): + errored = [ + { + "info": {"id": "m1", "role": "assistant", "error": {"message": "no such model"}}, + "parts": [], + } + ] + + result = await _run(monkeypatch, errored) + + assert result.is_error + assert "no such model" in (result.error_message or "") + + +async def test_execute_keeps_a_real_answer(monkeypatch): + answered = [ + { + "info": {"id": "m1", "role": "assistant", "tokens": {"input": 10, "output": 5}}, + "parts": [{"type": "text", "text": "clickhouse, prometheus"}], + } + ] + + result = await _run(monkeypatch, answered) + + assert not result.is_error + assert result.output == "clickhouse, prometheus" From 6050fa94671b69e994c6051167508f8a5e50a4f1 Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 19 Aug 2026 14:02:20 +0200 Subject: [PATCH 3/3] eval: read the provider error message out of its "data" wrapper The first CI run with the new guard surfaced the real payload: {"data": {"message": "The latest version of this model is only available hosted in China and requires explicit opt in: ...", "statusCode": ...}} _error_text only looked for message/detail/error at the top level, so this fell through to the raw-JSON fallback and got truncated at 300 chars. Look one level down under "data" as well, so the reason reaches the CI table intact. --- tests/eval/agent/opencode_agent.py | 13 +++++++++---- tests/eval/tests/test_agent_no_output.py | 2 ++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/eval/agent/opencode_agent.py b/tests/eval/agent/opencode_agent.py index c2a4a795..ada38d1a 100644 --- a/tests/eval/agent/opencode_agent.py +++ b/tests/eval/agent/opencode_agent.py @@ -97,10 +97,15 @@ def _free_port() -> int: def _error_text(error: Any) -> str: """Render opencode's error payload (shape varies by provider) for a message.""" if isinstance(error, dict): - for key in ("message", "detail", "error"): - value = error.get(key) - if isinstance(value, str) and value: - return value + # The human-readable text sits at the top level or one level down under + # "data", depending on which provider rejected the request. + for scope in (error, error.get("data")): + if not isinstance(scope, dict): + continue + for key in ("message", "detail", "error"): + value = scope.get(key) + if isinstance(value, str) and value: + return value return json.dumps(error)[:300] return str(error)[:300] diff --git a/tests/eval/tests/test_agent_no_output.py b/tests/eval/tests/test_agent_no_output.py index bb4f5bd9..d5afff0f 100644 --- a/tests/eval/tests/test_agent_no_output.py +++ b/tests/eval/tests/test_agent_no_output.py @@ -57,6 +57,8 @@ def test_turn_with_any_signal_is_accepted(final_text, tool_calls, tokens): ({"message": "boom"}, "boom"), ({"detail": "nested"}, "nested"), ({"code": 502}, '{"code": 502}'), + # The shape CI actually returned when deepseek-v4-flash became opt-in. + ({"data": {"message": "requires explicit opt in"}}, "requires explicit opt in"), ("plain", "plain"), ], )