ci: bump Go to 1.26.6, and make the eval harness fail loudly when the model is never invoked

Dockerfile

@@ -9,7 +9,7 @@
 # =============================================================================
 # Stage 1: Go builder
 # =============================================================================
-FROM golang:1.26-bookworm@sha256:5d2b868674b57c9e48cdd39e891acce4196b6926ca6d11e9c270a8f85106203d AS builder
+FROM golang:1.26-bookworm@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36 AS builder
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     git ca-certificates && \

go.mod

@@ -1,6 +1,6 @@
 module github.com/ethpandaops/panda
 
-go 1.26.5
+go 1.26.6
 
 require (
 	github.com/containerd/errdefs v1.0.0

tests/eval/agent/opencode_agent.py

@@ -94,6 +94,47 @@ def _free_port() -> int:
 # A single `opencode serve` is shared across all OpenCodeAgent instances with the
 # same config (keyed by the rendered opencode.json), so a pytest run with a
 # function-scoped agent fixture pays the server cold-start once, not per test.
+def _error_text(error: Any) -> str:
+    """Render opencode's error payload (shape varies by provider) for a message."""
+    if isinstance(error, dict):
+        # The human-readable text sits at the top level or one level down under
+        # "data", depending on which provider rejected the request.
+        for scope in (error, error.get("data")):
+            if not isinstance(scope, dict):
+                continue
+            for key in ("message", "detail", "error"):
+                value = scope.get(key)
+                if isinstance(value, str) and value:
+                    return value
+        return json.dumps(error)[:300]
+    return str(error)[:300]
+
+
+def _raise_if_no_output(
+    *,
+    provider_error: Any,
+    final_text: str,
+    tool_calls: list[ToolCallRecord],
+    tokens: int,
+    model: str,
+) -> None:
+    """Fail a turn that never reached the model, instead of scoring it as an answer.
+
+    A turn with no text, no tool call and no tokens means the model was never
+    invoked — a bad model id, an expired key, a provider outage. Left unreported
+    that grades as a wrong answer, so a broken harness looks like a quality
+    regression and the provider's retry path never fires.
+    """
+    if provider_error is not None:
+        raise RuntimeError(f"opencode provider error for {model}: {_error_text(provider_error)}")
+
+    if not final_text and not tool_calls and not tokens:
+        raise RuntimeError(
+            f"opencode produced no output for {model}: no text, no tool calls and no "
+            "tokens, so the model was never invoked (check the model id and API key)"
+        )
+
+
 _SHARED_SERVERS: dict[str, subprocess.Popen[bytes]] = {}
 _SHARED_URLS: dict[str, str] = {}
 _SHARED_CONTAINERS: dict[str, str] = {}  # server key -> docker container name (sandbox mode)
@@ -491,11 +532,18 @@ async def execute(
             input_tokens = 0
             output_tokens = 0
 
+            provider_error: Any = None
+
             for item in after:
                 d = self._as_dict(item)
                 info = d.get("info", {}) or {}
                 if info.get("id") in seen:
                     continue
+                # opencode reports a provider/model failure on the message itself.
+                # Without this the turn is indistinguishable from a legitimately
+                # empty answer, and a provider outage grades as a wrong answer.
+                if info.get("error") and provider_error is None:
+                    provider_error = info["error"]
                 if info.get("role") != "assistant":
                     continue
                 cost += float(info.get("cost") or 0.0)
@@ -521,6 +569,14 @@ async def execute(
                     elif ty == "text" and p.get("text"):
                         final_text = p["text"]
 
+            _raise_if_no_output(
+                provider_error=provider_error,
+                final_text=final_text,
+                tool_calls=tool_calls,
+                tokens=input_tokens + output_tokens,
+                model=f"{self.provider_id}/{self.model_id}",
+            )
+
             result.session_id = sid
             result.output = final_text
             result.tool_calls = tool_calls

tests/eval/tests/test_agent_no_output.py

@@ -0,0 +1,143 @@
+"""A turn that never reached the model must fail loudly, not score as an answer.
+
+CI hit this for real: every smoke case came back with 0 tokens, 0 tool calls and
+an empty answer, and the harness graded them as wrong answers (`crashed=False`,
+"0 errors"). That reads as an eval regression when it is actually an
+infrastructure failure, and it also skips the provider's retry-on-crash path.
+"""
+
+from typing import Any
+
+import pytest
+
+from agent.opencode_agent import OpenCodeAgent, _error_text, _raise_if_no_output
+from agent.wrapper import ToolCallRecord
+from config.settings import EvalSettings
+
+
+def test_provider_error_is_raised():
+    with pytest.raises(RuntimeError, match="provider error.*model not found"):
+        _raise_if_no_output(
+            provider_error={"message": "model not found"},
+            final_text="",
+            tool_calls=[],
+            tokens=0,
+            model="opencode-go/deepseek-v4-flash",
+        )
+
+
+def test_silent_empty_turn_is_raised():
+    with pytest.raises(RuntimeError, match="produced no output"):
+        _raise_if_no_output(
+            provider_error=None, final_text="", tool_calls=[], tokens=0, model="p/m"
+        )
+
+
+@pytest.mark.parametrize(
+    "final_text,tool_calls,tokens",
+    [
+        ("an answer", [], 0),  # answered, no tools
+        ("", [ToolCallRecord(name="t", input={})], 0),  # tools ran, no summary text
+        ("", [], 42),  # model billed tokens but said nothing
+    ],
+)
+def test_turn_with_any_signal_is_accepted(final_text, tool_calls, tokens):
+    _raise_if_no_output(
+        provider_error=None,
+        final_text=final_text,
+        tool_calls=tool_calls,
+        tokens=tokens,
+        model="p/m",
+    )
+
+
+@pytest.mark.parametrize(
+    "payload,expected",
+    [
+        ({"message": "boom"}, "boom"),
+        ({"detail": "nested"}, "nested"),
+        ({"code": 502}, '{"code": 502}'),
+        # The shape CI actually returned when deepseek-v4-flash became opt-in.
+        ({"data": {"message": "requires explicit opt in"}}, "requires explicit opt in"),
+        ("plain", "plain"),
+    ],
+)
+def test_error_text_renders_provider_shapes(payload, expected):
+    assert _error_text(payload) == expected
+
+
+class _FakeSession:
+    """Mimics an opencode session; the turn's messages only exist after chat().
+
+    execute() snapshots message ids before the turn and attributes only new ones,
+    so the pre-chat read must be empty or every message is filtered out as `seen`.
+    """
+
+    def __init__(self, messages: list[dict[str, Any]]) -> None:
+        self._messages = messages
+        self._chatted = False
+
+    async def create(self):
+        return type("S", (), {"id": "sess-1"})()
+
+    async def messages(self, id: str):  # noqa: A002 - matches the SDK signature
+        return self._messages if self._chatted else []
+
+    async def chat(self, **kwargs):
+        self._chatted = True
+        return None
+
+
+class _FakeClient:
+    def __init__(self, messages: list[dict[str, Any]]) -> None:
+        self.session = _FakeSession(messages)
+
+
+async def _run(monkeypatch, messages: list[dict[str, Any]]):
+    agent = OpenCodeAgent(EvalSettings())
+
+    async def _noop_ensure_server() -> None:
+        return None
+
+    monkeypatch.setattr(agent, "_ensure_server", _noop_ensure_server)
+    agent._client = _FakeClient(messages)
+    agent._langfuse = None
+    return await agent.execute("what datasources are available?")
+
+
+async def test_execute_marks_empty_turn_as_error(monkeypatch):
+    empty_turn = [{"info": {"id": "m1", "role": "assistant"}, "parts": []}]
+
+    result = await _run(monkeypatch, empty_turn)
+
+    assert result.is_error
+    assert "produced no output" in (result.error_message or "")
+    assert result.output == ""
+
+
+async def test_execute_surfaces_provider_error(monkeypatch):
+    errored = [
+        {
+            "info": {"id": "m1", "role": "assistant", "error": {"message": "no such model"}},
+            "parts": [],
+        }
+    ]
+
+    result = await _run(monkeypatch, errored)
+
+    assert result.is_error
+    assert "no such model" in (result.error_message or "")
+
+
+async def test_execute_keeps_a_real_answer(monkeypatch):
+    answered = [
+        {
+            "info": {"id": "m1", "role": "assistant", "tokens": {"input": 10, "output": 5}},
+            "parts": [{"type": "text", "text": "clickhouse, prometheus"}],
+        }
+    ]
+
+    result = await _run(monkeypatch, answered)
+
+    assert not result.is_error
+    assert result.output == "clickhouse, prometheus"