fix(proxy): retry missing response.created once

app/modules/proxy/_service/http_bridge/helpers.py

@@ -188,7 +188,7 @@
 
 logger = logging.getLogger("app.modules.proxy.service")
 _HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS = 5.0
-_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 240.0
+_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 30.0
 _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL = "missing_response_created_timeout"
 T = TypeVar("T")
 

app/modules/proxy/_service/http_bridge/upstream_events.py

@@ -423,23 +423,37 @@ async def _relay_http_bridge_upstream_messages(
                                 if not expired_owner:
                                     continue
                                 pending_count = len(session.pending_requests)
+                                can_retry_eventless_owner = pending_count == 1
+                            receive_cancelled = True
+                            if receive_task is not None:
+                                receive_cancelled = await _cancel_http_bridge_reader_child(
+                                    receive_task,
+                                    label="HTTP bridge upstream receive before missing response.created retry",
+                                )
+                                if receive_cancelled:
+                                    receive_task = None
+                            retried = False
+                            if can_retry_eventless_owner and receive_cancelled:
+                                try:
+                                    retried = await self._retry_http_bridge_precreated_request(session)
+                                except UpstreamWebSocketTransportError:
+                                    logger.warning(
+                                        "HTTP bridge missing response.created retry transport failed",
+                                        exc_info=True,
+                                    )
+                            if retried:
+                                continue
+                            async with session.pending_lock:
                                 for request_state in session.pending_requests:
                                     if request_state.failure_phase_override is None:
                                         request_state.failure_phase_override = "upstream"
                                     if request_state.failure_detail_override is None:
                                         request_state.failure_detail_override = (
                                             _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL
                                         )
-                            # Claim the session before cancelling receive so a
+                            # Claim the session before terminal settlement so a
                             # gate waiter cannot reopen this ambiguous socket.
                             session.closed = True
-                            if receive_task is not None:
-                                receive_cancelled = await _cancel_http_bridge_reader_child(
-                                    receive_task,
-                                    label="HTTP bridge upstream receive after missing response.created",
-                                )
-                                if receive_cancelled:
-                                    receive_task = None
                             _record_http_bridge_stuck_retire(
                                 reason=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL,
                                 session=session,

openspec/changes/retry-missing-response-created-once/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-07-25

openspec/changes/retry-missing-response-created-once/design.md

@@ -0,0 +1,90 @@
+## Context
+
+The HTTP Responses bridge records the monotonic time of each actual
+`response.create` send. If the request still owns the response-create gate and
+has no `response.created`, matched `response.*` event, response id, downstream
+sequence, or visible output, the upstream reader currently fails and retires
+the session after `min(stuck_gate_threshold, 240 seconds)`.
+
+The bridge already has a bounded `_retry_http_bridge_precreated_request` path.
+It permits at most one replay, rejects ambiguous response progress, preserves
+hard account ownership, protects account-scoped file references, and only
+strips a continuation anchor when the proxy retained a fingerprint-safe full
+resend body.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- Recover the production eventless failure before the client-safe timeout.
+- Reuse the existing replay and ownership rules instead of creating another
+  retry policy.
+- Settle and retire exactly once when recovery is not safe or does not work.
+- Keep missing acknowledgement neutral to account health.
+
+**Non-Goals:**
+
+- Recover streams that have matched any `response.*` lifecycle event.
+- Add durable cooldown or poison state across requests or replicas.
+- Retry more than once, extend the original request budget, or change public
+  response framing.
+
+## Decisions
+
+### 1. Use a 30-second acknowledgement window
+
+The eventless watchdog uses
+`min(http_responses_session_bridge_stuck_gate_retire_after_seconds, 30
+seconds)`, measured from the current send. Normal production TTFT is generally
+sub-second to low-single-digit seconds; 30 seconds leaves margin for transient
+startup delay while removing the four-minute dead period. Each real resend
+replaces the timestamp, so a replay gets one fresh acknowledgement window
+without extending the original request budget.
+
+### 2. Replay through the existing pre-created helper
+
+After eligibility is rechecked under lifecycle and pending-state locks, the
+reader cancels the old socket receive task and invokes the existing pre-created
+replay helper only when the eventless owner is the session's sole pending
+request. A successful reconnect/resend returns control to the same reader loop,
+which waits on the replacement socket while the downstream request stays open.
+
+The helper's existing `replay_count` bound makes this a single recovery
+attempt. Hard-affinity sessions reconnect on the same account. Continuations
+are replayed only from an explicitly retained retry-safe full-resend body, and
+file ownership continues to require the preferred account.
+
+### 3. Retire only after recovery is unavailable or exhausted
+
+If the helper declines replay, reconnect/resend fails, or the replacement send
+also reaches the deadline, the reader applies the existing
+`missing_response_created_timeout` overrides, records the stuck-retirement
+metric and terminal log, settles pending requests, and retires the bridge.
+Neither the retry nor terminal path marks the account unhealthy solely because
+the acknowledgement was missing.
+
+## Failure Modes
+
+- **The original send was accepted but its acknowledgement was lost.** Closing
+  the old socket discards any later output. Because no response lifecycle or
+  downstream-visible output was observed, client-side tools or other
+  downstream effects have not run; the bounded replay may spend extra upstream
+  compute but does not duplicate downstream effects.
+- **The request is continuity- or file-bound without safe replay evidence.**
+  The existing helper declines replay and the request fails closed at 30
+  seconds.
+- **Another request is pending on the same socket.** Reconnecting could orphan
+  that sibling's response, so the proxy skips replay and retains the existing
+  whole-session terminal cleanup.
+- **The reconnect or resend fails.** Existing typed retry errors are preserved
+  and the bridge is settled and retired exactly once.
+- **The replacement socket also stays silent.** `replay_count` blocks a second
+  replay; terminal cleanup runs after the replacement's 30-second window.
+
+## Example
+
+A request sends at monotonic time 1,000 and receives no matched response event.
+At 1,030 the reader cancels the old receive and safely resends once on a fresh
+socket. If `response.created` arrives at 1,032, the original downstream stream
+continues normally. If the fresh socket is still eventless at 1,060, the proxy
+returns the existing explicit timeout and retires the bridge.

openspec/changes/retry-missing-response-created-once/proposal.md

@@ -0,0 +1,42 @@
+## Why
+
+Current `main` bounds an HTTP bridge request that receives no
+`response.created` acknowledgement, but only after 240 seconds and by failing
+the client request. Production evidence on issue #1393 shows that an immediate
+fresh attempt commonly succeeds, so the proxy exposes a long avoidable failure
+instead of using its existing pre-visible replay path.
+
+## What Changes
+
+- Reduce the eventless pre-`response.created` watchdog cap from 240 seconds to
+  30 seconds.
+- On the first eventless timeout, cancel the old receive wait and attempt one
+  replay through the existing pre-created replay guards and fresh-socket
+  reconnect path.
+- Continue the original downstream stream when replay succeeds.
+- Preserve the current account-neutral terminal settlement and whole-session
+  retirement when replay is unsafe, reconnect/resend fails, or the replay also
+  misses `response.created`.
+- Keep hard-affinity and file-backed work on its required account and retain the
+  existing no-replay boundary after response lifecycle or downstream-visible
+  progress.
+
+## Capabilities
+
+### Modified Capabilities
+
+- `proxy-admission-control`: Recover one safely replayable eventless gate owner
+  before retiring the bridge.
+- `responses-api-compat`: Keep the retry transparent and bounded before any
+  response lifecycle or downstream-visible output.
+
+## Impact
+
+- Affected code: HTTP bridge eventless deadline and upstream-reader timeout
+  handling.
+- Affected surface: streaming Responses requests served through the HTTP to
+  upstream-WebSocket bridge.
+- No new setting, dependency, endpoint, schema, migration, account-health
+  penalty, or durable coordinator.
+- This partially addresses #1393. Cross-request cooldown and eventful
+  missing-created recovery remain separate work.

openspec/changes/retry-missing-response-created-once/specs/proxy-admission-control/spec.md

@@ -0,0 +1,111 @@
+## MODIFIED Requirements
+
+### Requirement: Stuck HTTP bridge response-create gate sessions are retired
+
+The proxy MUST retain the existing waiter-triggered retirement behavior for
+stale HTTP bridge response-create gate owners and MUST additionally enforce an
+owner-side deadline for a visible HTTP request whose current upstream
+`response.create` send remains completely eventless before `response.created`.
+The owner-side deadline MUST be measured from a monotonic timestamp recorded
+immediately before the current upstream send, MUST use the smaller of the
+configured stuck-gate retirement threshold and 30 seconds, MUST run without a
+second gate waiter, and MUST remain active when periodic SSE keepalives are
+disabled.
+
+The owner-side watchdog MUST apply only while the request owns the
+response-create gate, awaits `response.created`, has neither a response id nor
+recorded `response.created` latency, has received no matched `response.*`
+lifecycle event, and has produced no downstream-visible output or sequence
+evidence. Non-response telemetry such as `codex.rate_limits` MUST NOT suppress
+this watchdog. Any matched `response.*` lifecycle event, response-created
+milestone, or downstream-visible evidence MUST suppress the owner-side
+watchdog and leave existing timeout behavior unchanged.
+
+When the first owner-side deadline expires, the proxy MUST recheck eligibility,
+cancel the stale receive wait, and attempt one transparent replay only through
+the existing pre-created replay safety and ownership rules and only when the
+eventless owner is the session's sole pending request. Hard-affinity work MUST
+remain on the required account, account-scoped file ownership MUST be preserved,
+and a continuation MUST be replayed only from an explicitly retained retry-safe
+full-resend body. The retry MUST NOT extend the original request budget or mark
+the selected account unhealthy solely because `response.created` was missing.
+
+If replay is unsafe, reconnect/resend fails, or the replacement send reaches
+the deadline, the proxy MUST emit a structured low-cardinality timeout log and
+the existing stuck-retirement Prometheus counter, terminally settle every
+pending request exactly once, and retire the whole bridge session. It MUST NOT
+attempt a second replay.
+
+#### Scenario: Lone eventless gate owner recovers on a fresh socket
+
+- **GIVEN** a visible HTTP bridge request owns the response-create gate
+- **AND** its current send has no matched `response.*` event, response id, or
+  downstream-visible output
+- **WHEN** the smaller of the configured stuck threshold and 30 seconds elapses
+- **THEN** the proxy cancels the stale receive and safely replays the request at
+  most once on a fresh upstream socket
+- **AND** a successful replay continues the original downstream stream
+
+#### Scenario: A pending sibling prevents socket replacement
+
+- **GIVEN** an eventless gate owner reaches its deadline
+- **AND** another request is still pending on the same upstream socket
+- **WHEN** recovery is evaluated
+- **THEN** the proxy does not replace the socket for a transparent replay
+- **AND** it retains the existing whole-session terminal settlement
+
+#### Scenario: Send time rather than request age anchors each deadline
+
+- **GIVEN** a request spends most of its budget waiting for admission
+- **WHEN** the original request or its one replay sends `response.create`
+- **THEN** the owner-side deadline begins from that current send
+- **AND** prior admission time or the prior attempt does not make the send
+  immediately stale
+
+#### Scenario: Leading telemetry does not mask an eventless owner
+
+- **GIVEN** a pre-created gate owner receives `codex.rate_limits` but no matched
+  `response.*` lifecycle event
+- **WHEN** the owner-side deadline elapses
+- **THEN** the telemetry does not refresh or suppress the deadline
+- **AND** the proxy applies the same one-replay policy
+
+#### Scenario: Response lifecycle evidence suppresses the narrow watchdog
+
+- **GIVEN** a pre-created request receives any matched `response.*` lifecycle
+  event, response id, recorded `response.created` latency, or
+  downstream-visible output
+- **WHEN** the eventless owner-side deadline would otherwise elapse
+- **THEN** this watchdog does not reconnect or retire the session
+- **AND** existing stream, request-budget, and waiter-triggered behavior remains
+  authoritative
+
+#### Scenario: Unsafe or exhausted recovery fails closed
+
+- **GIVEN** an eventless pre-created owner reaches the owner-side deadline
+- **AND** safe replay is unavailable, fails, or has already been attempted
+- **WHEN** terminal cleanup runs
+- **THEN** every pending request is settled exactly once and the whole session
+  is retired
+- **AND** the selected account is not marked unhealthy solely because
+  `response.created` was missing
+- **AND** no second replay is attempted
+
+#### Scenario: Old pending work blocks a visible gate waiter
+
+- **WHEN** a visible HTTP bridge request receives
+  `response_create_gate_timeout`
+- **AND** at least one visible pending request on the same session is older than
+  the configured stuck-gate retirement threshold
+- **THEN** the proxy retires the bridge session so later requests can create a
+  fresh session
+- **AND** the waiter is rejected cleanly with `response_create_gate_timeout`
+
+#### Scenario: Healthy active stream is not retired during a normal wait
+
+- **WHEN** a visible HTTP bridge request times out waiting for the gate
+- **AND** the session has no pending visible request older than the configured
+  stuck-gate retirement threshold
+- **THEN** the proxy rejects only the waiter
+- **AND** the bridge session remains available for the existing in-flight
+  request