diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index c619878c0d..a080d1969f 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -188,7 +188,7 @@ logger = logging.getLogger("app.modules.proxy.service") _HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS = 5.0 -_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 240.0 +_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 30.0 _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL = "missing_response_created_timeout" T = TypeVar("T") diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 5fc06569f8..8041abae8a 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -423,6 +423,27 @@ async def _relay_http_bridge_upstream_messages( if not expired_owner: continue pending_count = len(session.pending_requests) + can_retry_eventless_owner = pending_count == 1 + receive_cancelled = True + if receive_task is not None: + receive_cancelled = await _cancel_http_bridge_reader_child( + receive_task, + label="HTTP bridge upstream receive before missing response.created retry", + ) + if receive_cancelled: + receive_task = None + retried = False + if can_retry_eventless_owner and receive_cancelled: + try: + retried = await self._retry_http_bridge_precreated_request(session) + except UpstreamWebSocketTransportError: + logger.warning( + "HTTP bridge missing response.created retry transport failed", + exc_info=True, + ) + if retried: + continue + async with session.pending_lock: for request_state in session.pending_requests: if request_state.failure_phase_override is None: request_state.failure_phase_override = "upstream" @@ -430,16 +451,9 @@ async def _relay_http_bridge_upstream_messages( request_state.failure_detail_override = ( _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL ) - # Claim the session before cancelling receive so a + # Claim the session before terminal settlement so a # gate waiter cannot reopen this ambiguous socket. session.closed = True - if receive_task is not None: - receive_cancelled = await _cancel_http_bridge_reader_child( - receive_task, - label="HTTP bridge upstream receive after missing response.created", - ) - if receive_cancelled: - receive_task = None _record_http_bridge_stuck_retire( reason=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, session=session, diff --git a/openspec/changes/retry-missing-response-created-once/.openspec.yaml b/openspec/changes/retry-missing-response-created-once/.openspec.yaml new file mode 100644 index 0000000000..3a038210f2 --- /dev/null +++ b/openspec/changes/retry-missing-response-created-once/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-25 diff --git a/openspec/changes/retry-missing-response-created-once/design.md b/openspec/changes/retry-missing-response-created-once/design.md new file mode 100644 index 0000000000..353be4e82a --- /dev/null +++ b/openspec/changes/retry-missing-response-created-once/design.md @@ -0,0 +1,90 @@ +## Context + +The HTTP Responses bridge records the monotonic time of each actual +`response.create` send. If the request still owns the response-create gate and +has no `response.created`, matched `response.*` event, response id, downstream +sequence, or visible output, the upstream reader currently fails and retires +the session after `min(stuck_gate_threshold, 240 seconds)`. + +The bridge already has a bounded `_retry_http_bridge_precreated_request` path. +It permits at most one replay, rejects ambiguous response progress, preserves +hard account ownership, protects account-scoped file references, and only +strips a continuation anchor when the proxy retained a fingerprint-safe full +resend body. + +## Goals / Non-Goals + +**Goals:** + +- Recover the production eventless failure before the client-safe timeout. +- Reuse the existing replay and ownership rules instead of creating another + retry policy. +- Settle and retire exactly once when recovery is not safe or does not work. +- Keep missing acknowledgement neutral to account health. + +**Non-Goals:** + +- Recover streams that have matched any `response.*` lifecycle event. +- Add durable cooldown or poison state across requests or replicas. +- Retry more than once, extend the original request budget, or change public + response framing. + +## Decisions + +### 1. Use a 30-second acknowledgement window + +The eventless watchdog uses +`min(http_responses_session_bridge_stuck_gate_retire_after_seconds, 30 +seconds)`, measured from the current send. Normal production TTFT is generally +sub-second to low-single-digit seconds; 30 seconds leaves margin for transient +startup delay while removing the four-minute dead period. Each real resend +replaces the timestamp, so a replay gets one fresh acknowledgement window +without extending the original request budget. + +### 2. Replay through the existing pre-created helper + +After eligibility is rechecked under lifecycle and pending-state locks, the +reader cancels the old socket receive task and invokes the existing pre-created +replay helper only when the eventless owner is the session's sole pending +request. A successful reconnect/resend returns control to the same reader loop, +which waits on the replacement socket while the downstream request stays open. + +The helper's existing `replay_count` bound makes this a single recovery +attempt. Hard-affinity sessions reconnect on the same account. Continuations +are replayed only from an explicitly retained retry-safe full-resend body, and +file ownership continues to require the preferred account. + +### 3. Retire only after recovery is unavailable or exhausted + +If the helper declines replay, reconnect/resend fails, or the replacement send +also reaches the deadline, the reader applies the existing +`missing_response_created_timeout` overrides, records the stuck-retirement +metric and terminal log, settles pending requests, and retires the bridge. +Neither the retry nor terminal path marks the account unhealthy solely because +the acknowledgement was missing. + +## Failure Modes + +- **The original send was accepted but its acknowledgement was lost.** Closing + the old socket discards any later output. Because no response lifecycle or + downstream-visible output was observed, client-side tools or other + downstream effects have not run; the bounded replay may spend extra upstream + compute but does not duplicate downstream effects. +- **The request is continuity- or file-bound without safe replay evidence.** + The existing helper declines replay and the request fails closed at 30 + seconds. +- **Another request is pending on the same socket.** Reconnecting could orphan + that sibling's response, so the proxy skips replay and retains the existing + whole-session terminal cleanup. +- **The reconnect or resend fails.** Existing typed retry errors are preserved + and the bridge is settled and retired exactly once. +- **The replacement socket also stays silent.** `replay_count` blocks a second + replay; terminal cleanup runs after the replacement's 30-second window. + +## Example + +A request sends at monotonic time 1,000 and receives no matched response event. +At 1,030 the reader cancels the old receive and safely resends once on a fresh +socket. If `response.created` arrives at 1,032, the original downstream stream +continues normally. If the fresh socket is still eventless at 1,060, the proxy +returns the existing explicit timeout and retires the bridge. diff --git a/openspec/changes/retry-missing-response-created-once/proposal.md b/openspec/changes/retry-missing-response-created-once/proposal.md new file mode 100644 index 0000000000..83c5d45268 --- /dev/null +++ b/openspec/changes/retry-missing-response-created-once/proposal.md @@ -0,0 +1,42 @@ +## Why + +Current `main` bounds an HTTP bridge request that receives no +`response.created` acknowledgement, but only after 240 seconds and by failing +the client request. Production evidence on issue #1393 shows that an immediate +fresh attempt commonly succeeds, so the proxy exposes a long avoidable failure +instead of using its existing pre-visible replay path. + +## What Changes + +- Reduce the eventless pre-`response.created` watchdog cap from 240 seconds to + 30 seconds. +- On the first eventless timeout, cancel the old receive wait and attempt one + replay through the existing pre-created replay guards and fresh-socket + reconnect path. +- Continue the original downstream stream when replay succeeds. +- Preserve the current account-neutral terminal settlement and whole-session + retirement when replay is unsafe, reconnect/resend fails, or the replay also + misses `response.created`. +- Keep hard-affinity and file-backed work on its required account and retain the + existing no-replay boundary after response lifecycle or downstream-visible + progress. + +## Capabilities + +### Modified Capabilities + +- `proxy-admission-control`: Recover one safely replayable eventless gate owner + before retiring the bridge. +- `responses-api-compat`: Keep the retry transparent and bounded before any + response lifecycle or downstream-visible output. + +## Impact + +- Affected code: HTTP bridge eventless deadline and upstream-reader timeout + handling. +- Affected surface: streaming Responses requests served through the HTTP to + upstream-WebSocket bridge. +- No new setting, dependency, endpoint, schema, migration, account-health + penalty, or durable coordinator. +- This partially addresses #1393. Cross-request cooldown and eventful + missing-created recovery remain separate work. diff --git a/openspec/changes/retry-missing-response-created-once/specs/proxy-admission-control/spec.md b/openspec/changes/retry-missing-response-created-once/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..02cf5329fc --- /dev/null +++ b/openspec/changes/retry-missing-response-created-once/specs/proxy-admission-control/spec.md @@ -0,0 +1,111 @@ +## MODIFIED Requirements + +### Requirement: Stuck HTTP bridge response-create gate sessions are retired + +The proxy MUST retain the existing waiter-triggered retirement behavior for +stale HTTP bridge response-create gate owners and MUST additionally enforce an +owner-side deadline for a visible HTTP request whose current upstream +`response.create` send remains completely eventless before `response.created`. +The owner-side deadline MUST be measured from a monotonic timestamp recorded +immediately before the current upstream send, MUST use the smaller of the +configured stuck-gate retirement threshold and 30 seconds, MUST run without a +second gate waiter, and MUST remain active when periodic SSE keepalives are +disabled. + +The owner-side watchdog MUST apply only while the request owns the +response-create gate, awaits `response.created`, has neither a response id nor +recorded `response.created` latency, has received no matched `response.*` +lifecycle event, and has produced no downstream-visible output or sequence +evidence. Non-response telemetry such as `codex.rate_limits` MUST NOT suppress +this watchdog. Any matched `response.*` lifecycle event, response-created +milestone, or downstream-visible evidence MUST suppress the owner-side +watchdog and leave existing timeout behavior unchanged. + +When the first owner-side deadline expires, the proxy MUST recheck eligibility, +cancel the stale receive wait, and attempt one transparent replay only through +the existing pre-created replay safety and ownership rules and only when the +eventless owner is the session's sole pending request. Hard-affinity work MUST +remain on the required account, account-scoped file ownership MUST be preserved, +and a continuation MUST be replayed only from an explicitly retained retry-safe +full-resend body. The retry MUST NOT extend the original request budget or mark +the selected account unhealthy solely because `response.created` was missing. + +If replay is unsafe, reconnect/resend fails, or the replacement send reaches +the deadline, the proxy MUST emit a structured low-cardinality timeout log and +the existing stuck-retirement Prometheus counter, terminally settle every +pending request exactly once, and retire the whole bridge session. It MUST NOT +attempt a second replay. + +#### Scenario: Lone eventless gate owner recovers on a fresh socket + +- **GIVEN** a visible HTTP bridge request owns the response-create gate +- **AND** its current send has no matched `response.*` event, response id, or + downstream-visible output +- **WHEN** the smaller of the configured stuck threshold and 30 seconds elapses +- **THEN** the proxy cancels the stale receive and safely replays the request at + most once on a fresh upstream socket +- **AND** a successful replay continues the original downstream stream + +#### Scenario: A pending sibling prevents socket replacement + +- **GIVEN** an eventless gate owner reaches its deadline +- **AND** another request is still pending on the same upstream socket +- **WHEN** recovery is evaluated +- **THEN** the proxy does not replace the socket for a transparent replay +- **AND** it retains the existing whole-session terminal settlement + +#### Scenario: Send time rather than request age anchors each deadline + +- **GIVEN** a request spends most of its budget waiting for admission +- **WHEN** the original request or its one replay sends `response.create` +- **THEN** the owner-side deadline begins from that current send +- **AND** prior admission time or the prior attempt does not make the send + immediately stale + +#### Scenario: Leading telemetry does not mask an eventless owner + +- **GIVEN** a pre-created gate owner receives `codex.rate_limits` but no matched + `response.*` lifecycle event +- **WHEN** the owner-side deadline elapses +- **THEN** the telemetry does not refresh or suppress the deadline +- **AND** the proxy applies the same one-replay policy + +#### Scenario: Response lifecycle evidence suppresses the narrow watchdog + +- **GIVEN** a pre-created request receives any matched `response.*` lifecycle + event, response id, recorded `response.created` latency, or + downstream-visible output +- **WHEN** the eventless owner-side deadline would otherwise elapse +- **THEN** this watchdog does not reconnect or retire the session +- **AND** existing stream, request-budget, and waiter-triggered behavior remains + authoritative + +#### Scenario: Unsafe or exhausted recovery fails closed + +- **GIVEN** an eventless pre-created owner reaches the owner-side deadline +- **AND** safe replay is unavailable, fails, or has already been attempted +- **WHEN** terminal cleanup runs +- **THEN** every pending request is settled exactly once and the whole session + is retired +- **AND** the selected account is not marked unhealthy solely because + `response.created` was missing +- **AND** no second replay is attempted + +#### Scenario: Old pending work blocks a visible gate waiter + +- **WHEN** a visible HTTP bridge request receives + `response_create_gate_timeout` +- **AND** at least one visible pending request on the same session is older than + the configured stuck-gate retirement threshold +- **THEN** the proxy retires the bridge session so later requests can create a + fresh session +- **AND** the waiter is rejected cleanly with `response_create_gate_timeout` + +#### Scenario: Healthy active stream is not retired during a normal wait + +- **WHEN** a visible HTTP bridge request times out waiting for the gate +- **AND** the session has no pending visible request older than the configured + stuck-gate retirement threshold +- **THEN** the proxy rejects only the waiter +- **AND** the bridge session remains available for the existing in-flight + request diff --git a/openspec/changes/retry-missing-response-created-once/specs/responses-api-compat/spec.md b/openspec/changes/retry-missing-response-created-once/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..fde6f6d6d1 --- /dev/null +++ b/openspec/changes/retry-missing-response-created-once/specs/responses-api-compat/spec.md @@ -0,0 +1,104 @@ +## MODIFIED Requirements + +### Requirement: Failed precreated HTTP bridge replay retires stale sessions + +When an HTTP bridge request is still pending before upstream +`response.completed` and the upstream websocket closes or times out before the +pending request can be completed, the service MUST fail the pending request +terminally and retire the affected bridge session if precreated replay does not +reconnect and resend successfully. + +For an eventless `response.create` that reaches the owner-side +missing-`response.created` deadline, the service MUST cancel the old receive +wait and MAY transparently replay once only when the existing pre-created +replay predicate proves there is no matched response lifecycle, upstream model +output, downstream sequence, or downstream-visible output and the eventless +owner is the session's sole pending request. The replay MUST preserve +hard-affinity and account-scoped file ownership. A request carrying +`previous_response_id` MUST fail closed unless the proxy retained an explicitly +retry-safe full-resend body that can be replayed without the anchor. + +If that replay succeeds, the original downstream stream MUST continue without +a terminal event. If replay is ineligible, reconnect/resend fails, or the +replacement send also times out, the service MUST terminally settle the request +and retire the session exactly once without an account-health penalty for the +missing acknowledgement. + +#### Scenario: First eventless timeout recovers transparently + +- **GIVEN** an HTTP bridge request has no response lifecycle or visible progress +- **AND** its first `response.create` reaches the missing-created deadline +- **WHEN** the existing replay guards accept the request and reconnect/resend + succeeds +- **THEN** the service continues reading the replacement upstream socket +- **AND** the downstream stream receives no terminal failure for the first + timeout + +#### Scenario: Continuity without a safe full resend is not replayed + +- **GIVEN** an eventless HTTP bridge request carries `previous_response_id` +- **AND** the proxy has no explicitly retry-safe full-resend body +- **WHEN** the missing-created deadline elapses +- **THEN** the service does not replay the continuation +- **AND** it terminally settles the request and retires the bridge + +#### Scenario: Replay preserves hard account and file ownership + +- **GIVEN** an eventless request has hard affinity or an account-scoped file +- **WHEN** it is eligible for the one transparent replay +- **THEN** the replacement connection uses the required owner account +- **AND** the file reference is not moved to an account that does not own it + +#### Scenario: Pending sibling blocks transparent replay + +- **GIVEN** an eventless request shares its upstream socket with another pending + request +- **WHEN** the missing-created deadline elapses +- **THEN** the service does not replace the shared socket for replay +- **AND** it terminally settles the stale bridge through existing cleanup + +#### Scenario: Replacement timeout is terminal + +- **GIVEN** an eventless request was replayed once on a fresh upstream socket +- **WHEN** the replacement send also misses `response.created` +- **THEN** the service does not replay again +- **AND** it emits one terminal failure and retires the bridge session + +#### Scenario: Precreated replay fails after upstream disconnect + +- **WHEN** an HTTP bridge request is pending before `response.completed` +- **AND** the upstream websocket closes before the request completes +- **AND** precreated replay fails to reconnect and resend the request +- **THEN** the pending request is removed from the bridge queue +- **AND** the per-session response-create gate is released +- **AND** the bridge session is closed and removed from local reuse +- **AND** the terminal error preserves the original failure code such as + `stream_incomplete` or `upstream_request_timeout` + +#### Scenario: Terminal logging failure does not preserve stale bridge ownership + +- **WHEN** a failed pending HTTP bridge request is being logged as terminal +- **AND** request-log writing fails +- **THEN** the service still removes the stale bridge session from local reuse +- **AND** the service releases any durable bridge ownership for that stale + session + +#### Scenario: Concurrent waiter cannot submit on retired stale bridge + +- **WHEN** an HTTP bridge request is waiting on a session response-create gate +- **AND** the upstream reader retires that same bridge session after a failed + precreated replay +- **THEN** the waiting request or prewarm is rejected before it is appended to + pending requests or sent upstream +- **AND** the retired bridge session remains closed and removed from local reuse +- **AND** the post-admission ownership check, pending enqueue, and upstream send + are mutually exclusive with stale-session retirement + +#### Scenario: Unregistered stale bridge reference cannot submit after admission + +- **WHEN** an HTTP bridge request or prewarm holds a stale bridge session + reference +- **AND** that bridge session is no longer the registered local owner for its + session key +- **THEN** the request is rejected after response-create gate admission and + before it is appended or sent upstream diff --git a/openspec/changes/retry-missing-response-created-once/tasks.md b/openspec/changes/retry-missing-response-created-once/tasks.md new file mode 100644 index 0000000000..e5ac8357e5 --- /dev/null +++ b/openspec/changes/retry-missing-response-created-once/tasks.md @@ -0,0 +1,21 @@ +## 1. Specification + +- [x] 1.1 Define the bounded eventless retry and terminal fallback contracts. +- [x] 1.2 Strictly validate the OpenSpec change. + +## 2. Implementation + +- [x] 2.1 Reduce the eventless acknowledgement cap to 30 seconds. +- [x] 2.2 Cancel the stale receive and invoke one existing safe pre-created + replay before terminal settlement. +- [x] 2.3 Preserve hard affinity, file ownership, request budget, account + neutrality, and whole-session retirement on exhaustion. + +## 3. Verification + +- [x] 3.1 Add regressions for first-timeout recovery, telemetry-only silence, + unsafe replay, and second-timeout settlement. +- [x] 3.2 Run focused bridge tests, lint, format, type, architecture, and strict + OpenSpec validation. +- [x] 3.3 Review the final diff for replay widening, duplicate settlement, + affinity movement, account penalties, and unrelated edits. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 1d4058c434..8fba4b724b 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -126,6 +126,44 @@ def _make_eventless_http_bridge_owner( ) +class _SilentTrackingUpstream: + def __init__(self, *, leading_telemetry: bool = False) -> None: + self.leading_telemetry = leading_telemetry + self.telemetry_emitted = False + self.blocking_receive_started = asyncio.Event() + self.receive_cancellations = 0 + self.closed = False + self.sent_texts: list[str] = [] + + async def receive(self) -> UpstreamWebSocketMessage: + if self.leading_telemetry and not self.telemetry_emitted: + self.telemetry_emitted = True + return UpstreamWebSocketMessage( + kind="text", + text=json.dumps( + { + "type": "codex.rate_limits", + "plan_type": "pro", + "rate_limits": {"allowed": True, "limit_reached": False}, + }, + separators=(",", ":"), + ), + ) + self.blocking_receive_started.set() + try: + await asyncio.Event().wait() + raise AssertionError("unreachable") + except asyncio.CancelledError: + self.receive_cancellations += 1 + raise + + async def send_text(self, text: str) -> None: + self.sent_texts.append(text) + + async def close(self) -> None: + self.closed = True + + def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_safe_cap() -> None: request_state = _make_eventless_http_bridge_owner() @@ -134,7 +172,7 @@ def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_ request_state, stuck_gate_retire_after_seconds=300.0, ) - == 340.0 + == 130.0 ) assert ( http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( @@ -150,7 +188,7 @@ def test_http_bridge_eventless_precreated_deadline_uses_current_send_and_client_ request_state, stuck_gate_retire_after_seconds=300.0, ) - == 340.0 + == 130.0 ) @@ -17374,6 +17412,176 @@ async def close(self) -> None: assert "http_bridge_event event=missing_response_created_timeout" in caplog.text +@pytest.mark.asyncio +@pytest.mark.parametrize("leading_telemetry", [False, True], ids=["silent", "leading-telemetry"]) +async def test_http_bridge_eventless_timeout_retries_once_on_fresh_same_account_socket( + monkeypatch: pytest.MonkeyPatch, + leading_telemetry: bool, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + old_upstream = _SilentTrackingUpstream(leading_telemetry=leading_telemetry) + replacement_upstream = _SilentTrackingUpstream() + session = _make_bridge_session(key_value=f"eventless-retry-{leading_telemetry}") + session.upstream = cast(UpstreamResponsesWebSocket, old_upstream) + service._http_bridge_sessions[session.key] = session + settings = _make_app_settings( + stream_idle_timeout_seconds=60.0, + http_responses_session_bridge_request_budget_seconds=60.0, + http_responses_session_bridge_stuck_gate_retire_after_seconds=0.1, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + record_stuck_retire = Mock() + monkeypatch.setattr(proxy_service, "_record_http_bridge_stuck_retire", record_stuck_retire) + fail_reader = AsyncMock() + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + gate = session.response_create_gate + await gate.acquire() + owner = _make_eventless_http_bridge_owner( + request_id=f"req-eventless-retry-{leading_telemetry}", + sent_at=time.monotonic() if leading_telemetry else time.monotonic() - 1.0, + ) + owner.started_at = time.monotonic() + owner.response_create_gate = gate + owner.request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}' + owner.preferred_account_id = session.account.id + owner.account_response_create_lease = cast(Any, object()) + async with session.pending_lock: + session.pending_requests.append(owner) + session.queued_request_count = 1 + + async def reconnect( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + assert target_session is session + assert kwargs["request_state"] is owner + assert kwargs["require_same_account"] is True + owner.response_create_sent_at = None + target_session.upstream = cast(UpstreamResponsesWebSocket, replacement_upstream) + + reconnect_mock = AsyncMock(side_effect=reconnect) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect_mock) + + reader_task = asyncio.create_task(service._relay_http_bridge_upstream_messages(session)) + try: + await asyncio.wait_for(replacement_upstream.blocking_receive_started.wait(), timeout=1.0) + + reconnect_mock.assert_awaited_once_with( + session, + request_state=owner, + require_same_account=True, + ) + assert replacement_upstream.sent_texts == [owner.request_text] + assert old_upstream.receive_cancellations == 1 + assert owner.replay_count == 1 + assert owner.response_create_sent_at is not None + assert owner.failure_detail_override is None + assert list(session.pending_requests) == [owner] + assert session.closed is False + assert owner.event_queue is not None + if leading_telemetry: + telemetry_block = owner.event_queue.get_nowait() + assert telemetry_block is not None + assert '"type":"codex.rate_limits"' in telemetry_block + assert owner.event_queue.empty() is True + fail_reader.assert_not_awaited() + record_stuck_retire.assert_not_called() + finally: + reader_task.cancel() + with pytest.raises(asyncio.CancelledError): + await reader_task + + assert replacement_upstream.receive_cancellations == 1 + + +@pytest.mark.asyncio +async def test_http_bridge_eventless_retry_second_timeout_settles_and_retires_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + old_upstream = _SilentTrackingUpstream() + replacement_upstream = _SilentTrackingUpstream() + session = _make_bridge_session(key_value="eventless-retry-exhausted") + session.upstream = cast(UpstreamResponsesWebSocket, old_upstream) + service._http_bridge_sessions[session.key] = session + settings = _make_app_settings( + stream_idle_timeout_seconds=60.0, + http_responses_session_bridge_request_budget_seconds=60.0, + http_responses_session_bridge_stuck_gate_retire_after_seconds=0.02, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + write_request_log = AsyncMock() + monkeypatch.setattr(service, "_write_request_log", write_request_log) + record_stuck_retire = Mock() + monkeypatch.setattr(proxy_service, "_record_http_bridge_stuck_retire", record_stuck_retire) + original_fail_reader = service._fail_http_bridge_reader_and_maybe_retire + fail_reader = AsyncMock(wraps=original_fail_reader) + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + gate = session.response_create_gate + await gate.acquire() + owner = _make_eventless_http_bridge_owner( + request_id="req-eventless-retry-exhausted", + sent_at=time.monotonic() - 1.0, + ) + owner.started_at = time.monotonic() + owner.response_create_gate = gate + owner.request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}' + owner.preferred_account_id = session.account.id + owner.account_response_create_lease = cast(Any, object()) + event_queue = owner.event_queue + assert event_queue is not None + async with session.pending_lock: + session.pending_requests.append(owner) + session.queued_request_count = 1 + + async def reconnect( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + assert target_session is session + assert kwargs["request_state"] is owner + assert kwargs["require_same_account"] is True + owner.response_create_sent_at = None + target_session.upstream = cast(UpstreamResponsesWebSocket, replacement_upstream) + + reconnect_mock = AsyncMock(side_effect=reconnect) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect_mock) + + await asyncio.wait_for(service._relay_http_bridge_upstream_messages(session), timeout=1.0) + + event_blocks: list[str] = [] + while (event_block := await asyncio.wait_for(event_queue.get(), timeout=0.1)) is not None: + event_blocks.append(event_block) + assert len(event_blocks) == 1 + assert '"code":"upstream_request_timeout"' in event_blocks[0] + assert event_queue.empty() is True + reconnect_mock.assert_awaited_once_with( + session, + request_state=owner, + require_same_account=True, + ) + assert replacement_upstream.sent_texts == [owner.request_text] + assert old_upstream.receive_cancellations == 1 + assert replacement_upstream.receive_cancellations == 1 + assert owner.replay_count == 1 + assert owner.failure_detail_override == "missing_response_created_timeout" + assert list(session.pending_requests) == [] + assert session.queued_request_count == 0 + assert session.closed is True + assert session.key not in service._http_bridge_sessions + assert gate.locked() is False + assert write_request_log.await_count == 1 + fail_reader.assert_awaited_once() + assert fail_reader.await_args.kwargs["penalize_account"] is False + record_stuck_retire.assert_called_once_with( + reason="missing_response_created_timeout", + session=session, + ) + + @pytest.mark.asyncio async def test_http_bridge_eventless_timeout_yields_to_locked_send_failure_cleanup( monkeypatch: pytest.MonkeyPatch,