WIP: feat(eve): capture gateway correlation details on model-call failures

.changeset/gateway-stream-termination-diagnostics.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Model-call failures now surface AI Gateway correlation details — `generationId`, plus the `provider` and `model` actually routed to — in both `step.failed` details and logs, including for failures the semantic-error catalog does not recognize (those previously logged a raw inspector dump with no structured fields). `generationId` is the join key for looking up the upstream cause in gateway telemetry. A gateway stream that ends before its terminal chunk is also now treated as transient, so it retries with backoff instead of parking the session on first contact.

packages/eve/src/harness/model-call-error.test.ts

@@ -77,6 +77,25 @@ function gatewayModelCallError(input: {
   return error;
 }
 
+/**
+ * Mirrors the frame AI Gateway synthesizes when a provider stream ends without
+ * a terminal chunk: a plain-object `cause` keyed by `code`/`origin` with
+ * snake_case correlation fields, and no status code.
+ */
+function streamTerminatedModelCallError(): Error {
+  return new Error("Upstream stream ended before terminal chunk", {
+    cause: {
+      code: "gateway_stream_terminated",
+      generation_id: "gen_01KYWVKVSYAH1XKJ08CG9VPB4A",
+      message: "Upstream stream ended before terminal chunk",
+      model: "alibaba:glm-5.2",
+      origin: "gateway",
+      provider: "alibaba",
+      upstream_finish_received: false,
+    },
+  });
+}
+
 function directApiCallError(input: {
   readonly data?: Record<string, unknown>;
   readonly message?: string;
@@ -175,6 +194,13 @@ describe("classifyModelCallError", () => {
     expect(classifyModelCallError(err)).toBe("retry");
   });
 
+  it("returns retry for a gateway stream termination", () => {
+    // The frame carries no status code and is not marked retryable, so before
+    // the `gateway-stream-terminated` catalog rule this fell through to
+    // "recoverable" and parked the turn without a single retry.
+    expect(classifyModelCallError(streamTerminatedModelCallError())).toBe("retry");
+  });
+
   it("returns retry for Anthropic overloaded stream payloads", () => {
     const overloaded = {
       message: "Overloaded",
@@ -607,4 +633,16 @@ describe("extractModelCallErrorDetails", () => {
       upstreamType: "invalid_request_error",
     });
   });
+
+  it("lifts correlation fields off a synthesized gateway stream-termination frame", () => {
+    // The frame carries `code` + `origin` and snake_case fields rather than a
+    // `Gateway*` class name, so it exercises the shape-detection path.
+    const details = extractModelCallErrorDetails(streamTerminatedModelCallError());
+
+    expect(details).toMatchObject({
+      generationId: "gen_01KYWVKVSYAH1XKJ08CG9VPB4A",
+      model: "alibaba:glm-5.2",
+      provider: "alibaba",
+    });
+  });
 });

packages/eve/src/harness/model-call-error.ts

@@ -35,7 +35,9 @@ export interface UpstreamRejectionSummary {
 interface ModelCallErrorSignals {
   readonly apiCallError: boolean;
   readonly apiErrorMessage?: string;
+  readonly gatewayModel?: string;
   readonly gatewayName?: string;
+  readonly gatewayProvider?: string;
   readonly gatewayType?: string;
   readonly generationId?: string;
   readonly responseBodySnippet?: string;
@@ -173,6 +175,11 @@ export function extractModelCallErrorDetails(error: unknown): JsonObject {
   appendJsonField(details, "gatewayType", signals.gatewayType);
   appendJsonField(details, "statusCode", signals.statusCode);
   appendJsonField(details, "generationId", signals.generationId);
+  // Which upstream the gateway actually routed to. A canonical slug like
+  // `zai/glm-5.2` is served by many providers, so the failing one is not
+  // inferable from the model id alone.
+  appendJsonField(details, "provider", signals.gatewayProvider);
+  appendJsonField(details, "model", signals.gatewayModel);
   appendJsonField(details, "upstreamStatusCode", signals.upstreamStatusCode);
   appendJsonField(details, "upstreamType", signals.upstreamType);
   appendJsonField(details, "upstreamMessage", signals.upstreamMessage);
@@ -341,9 +348,16 @@ function readModelCallErrorSignals(error: unknown): ModelCallErrorSignals {
     apiErrorMessage:
       upstreamBody?.apiErrorMessage ??
       firstInformativeApiMessage([readErrorMessage(upstreamError)]),
+    // Synthesized gateway stream frames use snake_case; structured gateway
+    // errors use camelCase. Read both rather than guessing which shape arrived.
+    gatewayModel: readStringField(gatewayError, "model"),
     gatewayName: readErrorName(gatewayError),
+    gatewayProvider: readStringField(gatewayError, "provider"),
     gatewayType: readStringField(gatewayError, "type"),
-    generationId: readStringField(gatewayError, "generationId") ?? upstreamBody?.generationId,
+    generationId:
+      readStringField(gatewayError, "generationId") ??
+      readStringField(gatewayError, "generation_id") ??
+      upstreamBody?.generationId,
     responseBodySnippet:
       responseBody === undefined
         ? undefined
@@ -360,7 +374,17 @@ function findGatewayError(error: unknown): unknown {
   for (const candidate of walkCauseChain(error)) {
     const name = readErrorName(candidate);
     const type = readStringField(candidate, "type");
-    if (name?.startsWith("Gateway") || type?.endsWith("_error") || type === "rate_limit_exceeded") {
+    if (
+      name?.startsWith("Gateway") ||
+      type?.endsWith("_error") ||
+      type === "rate_limit_exceeded" ||
+      // Gateway-synthesized stream frames (e.g. `gateway_stream_terminated`)
+      // carry `code` + `origin` instead of a `Gateway*` class name or a
+      // `*_error` type, so they would otherwise never be found here — and the
+      // correlation fields they carry would be dropped.
+      (readStringField(candidate, "code")?.startsWith("gateway_") === true &&
+        readStringField(candidate, "origin") === "gateway")
+    ) {
       return candidate;
     }
   }

packages/eve/src/harness/semantic-errors/rules/gateway.ts

@@ -1,4 +1,12 @@
-import { allOf, anyOf, messageMatches, nameIs, typeIs, type SemanticErrorRule } from "../rule.js";
+import {
+  allOf,
+  anyOf,
+  codeIs,
+  messageMatches,
+  nameIs,
+  typeIs,
+  type SemanticErrorRule,
+} from "../rule.js";
 
 /** The summary `name` shared by the gateway-auth rule variants. */
 const GATEWAY_AUTH_FAILURE_SUMMARY_NAME = "AI Gateway authentication failed";
@@ -81,4 +89,19 @@ export const GATEWAY_RULES: readonly SemanticErrorRule[] = [
     message: "The model provider is overloaded or timing out upstream of AI Gateway.",
     hint: "This is transient — retry shortly, or switch models with `/model` in `eve dev`.",
   },
+  {
+    // The gateway synthesizes this frame when a provider stream ends without a
+    // terminal chunk — a dropped connection, or a mid-stream throttle the
+    // provider raised after the response was already committed (so the gateway
+    // could no longer fail over). It is a `code`, not a `type`, and carries no
+    // status code, so without this rule it fell through `classifyModelCallError`
+    // to `recoverable` and parked the turn on first contact with zero retries.
+    // `transient` is what earns it the standard 3 attempts with backoff.
+    id: "gateway-stream-terminated",
+    name: "Model stream ended early",
+    tags: ["gateway", "transient"],
+    when: codeIs("gateway_stream_terminated"),
+    message: "The model provider's stream ended before completing.",
+    hint: "Usually a transient upstream throttle or dropped connection — retries are automatic. If it persists, take the `generationId` from the failure details to AI Gateway telemetry for the upstream cause.",
+  },
 ];

packages/eve/src/harness/semantic-errors/semantic-errors.test.ts

@@ -175,6 +175,18 @@ describe("summarizeKnownError (catalog table)", () => {
       error: new TypeError("terminated"),
       id: "network-request-failed",
     },
+    {
+      title: "gateway stream terminated before a terminal chunk",
+      error: new Error("Upstream stream ended before terminal chunk", {
+        cause: {
+          code: "gateway_stream_terminated",
+          message: "Upstream stream ended before terminal chunk",
+          origin: "gateway",
+          upstream_finish_received: false,
+        },
+      }),
+      id: "gateway-stream-terminated",
+    },
   ];
 
   for (const testCase of cases) {

packages/eve/src/harness/tool-loop.test.ts

@@ -3560,6 +3560,36 @@ describe("createToolLoopHarness", () => {
     );
   });
 
+  it("logs correlation details even when the failure is unrecognized", async () => {
+    // Regression guard: log fields used to carry `details` only for recognized
+    // failures, so `generationId` and friends were dropped on exactly the
+    // failures that are hardest to diagnose.
+    const unrecognized = Object.assign(new Error("something we have no rule for"), {
+      generationId: "gen_tool_loop",
+      name: "GatewayWeirdNewError",
+      statusCode: 418,
+    });
+    setupMockAgentError(unrecognized);
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    try {
+      const { emit } = createEventCollector();
+      const runStep = createToolLoopHarness(createTestConfig("conversation", emit));
+      await runStep(createTestSession(), { message: "Hi" });
+
+      const logged = errorSpy.mock.calls.find(
+        ([, fields]) => (fields as { details?: unknown } | undefined)?.details !== undefined,
+      );
+      expect(logged).toBeDefined();
+      expect((logged![1] as { details: Record<string, unknown> }).details).toMatchObject({
+        generationId: "gen_tool_loop",
+        statusCode: 418,
+      });
+    } finally {
+      errorSpy.mockRestore();
+    }
+  });
+
   it("emits the full terminal failure cascade on a structural 4xx model-call error", async () => {
     // 400/401/403/404 responses are classified as terminal — the
     // session is torn down because retrying would hit the same wall.

packages/eve/src/harness/tool-loop.ts

@@ -1470,13 +1470,16 @@ function buildModelCallFailureDetails(input: {
 }
 
 /**
- * Builds the structured log fields for a model-call failure. When the
- * failure was recognized (catalog match or extracted upstream rejection),
- * attach the compact `details` payload and *omit* the raw `error` so the
- * logger's `util.inspect` of the cause chain (which would render
- * `[object Object]` for upstream `APICallError` shapes) is bypassed.
- * Otherwise fall back to the raw error so unrecognized failures keep
- * their full stack in logs.
+ * Builds the structured log fields for a model-call failure.
+ *
+ * `details` rides along whenever it has anything in it, recognized or not:
+ * these are the correlation fields (`generationId`, `provider`,
+ * `upstreamError`, …) an operator needs to take a failure to gateway
+ * telemetry, and they used to be dropped on exactly the unrecognized failures
+ * that are hardest to diagnose. Recognized failures still omit the raw `error`
+ * so the logger's `util.inspect` of the cause chain — which renders
+ * `[object Object]` for upstream `APICallError` shapes — is bypassed;
+ * unrecognized ones keep it for the full stack.
  */
 function buildModelCallFailureLogFields(input: {
   readonly error: unknown;
@@ -1490,9 +1493,12 @@ function buildModelCallFailureLogFields(input: {
     errorId: input.errorId,
     sessionId: input.sessionId,
     turnId: input.turnId,
+    ...(Object.keys(input.modelCallDetails).length > 0 && {
+      details: input.modelCallDetails,
+    }),
   };
   if (input.recognized) {
-    return { ...base, details: input.modelCallDetails };
+    return base;
   }
   return { ...base, error: input.error };
 }