fix(proxy): hold fenced hard turns through cooldown

.all-contributorsrc

@@ -1232,6 +1232,16 @@
         "code",
         "test"
       ]
+    },
+    {
+      "login": "kevinsslin",
+      "name": "Kevin Lin",
+      "avatar_url": "https://avatars.githubusercontent.com/u/86810837?v=4",
+      "profile": "https://github.com/kevinsslin",
+      "contributions": [
+        "code",
+        "test"
+      ]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -286,6 +286,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/kidclone3"><img src="https://avatars.githubusercontent.com/u/54184969?v=4?s=100" width="100px;" alt="DuyBui"/><br /><sub><b>DuyBui</b></sub></a><br /><a href="https://github.com/Soju06/codex-lb/commits?author=kidclone3" title="Code">💻</a> <a href="https://github.com/Soju06/codex-lb/commits?author=kidclone3" title="Tests">⚠️</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kevinsslin"><img src="https://avatars.githubusercontent.com/u/86810837?v=4?s=100" width="100px;" alt="Kevin Lin"/><br /><sub><b>Kevin Lin</b></sub></a><br /><a href="https://github.com/Soju06/codex-lb/commits?author=kevinsslin" title="Code">💻</a> <a href="https://github.com/Soju06/codex-lb/commits?author=kevinsslin" title="Tests">⚠️</a></td>
     </tr>
   </tbody>
 </table>

app/modules/proxy/_service/http_bridge/streaming.py

@@ -3642,10 +3642,149 @@ async def retry_precreated_for_idle_recovery(
                     ),
                 )
 
+        def operation_fenced_cooldown_wait_enabled() -> bool:
+            """Allow a hard turn to wait until its durable fence can arbitrate recovery."""
+            return (
+                getattr(
+                    _service_get_settings(),
+                    "http_responses_session_bridge_ambiguous_continuation_recovery_mode",
+                    "fail_closed",
+                )
+                in {"server_anchored_replay_once", "server_indefinite_recovery"}
+                and getattr(_service_get_settings(), "http_responses_session_bridge_operation_ledger_enabled", True)
+                and request_state.hard_continuity_anchor
+                and session.durable_session_id is not None
+                and session.durable_owner_epoch is not None
+                and request_state.previous_response_id is None
+                and request_state.response_id is None
+                and request_state.response_event_count == 0
+            )
+
         def continuity_bound_without_safe_replay() -> bool:
             """Do not hold a client stream through a cooldown we cannot use."""
             return _http_bridge_continuity_bound_without_safe_replay(request_state) and not (
-                _http_bridge_server_anchored_replay_enabled(request_state)
+                _http_bridge_server_anchored_replay_enabled(request_state) or operation_fenced_cooldown_wait_enabled()
+            )
+
+        async def wait_through_operation_fenced_startup_cooldown() -> bool:
+            if session.key.strength != "hard" or not operation_fenced_cooldown_wait_enabled():
+                return False
+            retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds(session)
+            if retry_cooldown_seconds <= 0:
+                return False
+            remaining_budget_seconds = request_deadline - _service_time().monotonic()
+            if remaining_budget_seconds <= 0:
+                return False
+            wait_seconds = min(retry_cooldown_seconds, remaining_budget_seconds)
+            async with session.pending_lock:
+                if session.queued_request_count >= queue_limit:
+                    raise ProxyResponseError(
+                        429,
+                        openai_error(
+                            "bridge_queue_full",
+                            "HTTP responses session bridge queue is full",
+                            error_type="rate_limit_error",
+                        ),
+                    )
+                session.queued_request_count += 1
+            _log_http_bridge_event(
+                "wait_operation_fenced_cooldown",
+                session.key,
+                account_id=session.account.id,
+                model=session.request_model,
+                detail="hard_turn_operation_fence",
+                cache_key_family=session.key.affinity_kind,
+            )
+            logger.info(
+                "HTTP bridge waiting through retry-circuit cooldown before durable hard-turn arbitration "
+                "request_id=%s wait_seconds=%.1f remaining_budget_seconds=%.1f",
+                request_state.request_id,
+                wait_seconds,
+                remaining_budget_seconds,
+            )
+            # No upstream request has been dispatched on this path.  After the
+            # cooldown, normal submission still has to create or claim the
+            # durable operation fence before response.create can be sent.
+            try:
+                current_instance = _service_get_settings().http_responses_session_bridge_instance_id
+                lease_refresh_interval_seconds = max(
+                    1.0,
+                    min(
+                        _http_bridge_durable_lease_ttl_seconds() / 3.0,
+                        wait_seconds,
+                    ),
+                )
+                remaining_wait_seconds = wait_seconds
+                while remaining_wait_seconds > 0:
+                    sleep_seconds = min(remaining_wait_seconds, lease_refresh_interval_seconds)
+                    await asyncio.sleep(sleep_seconds)
+                    remaining_wait_seconds = max(0.0, remaining_wait_seconds - sleep_seconds)
+                    if remaining_wait_seconds <= 0:
+                        break
+                    try:
+                        owner_lookup = await self._durable_bridge.renew_live_session(
+                            session_id=session.durable_session_id,
+                            api_key_id=session.key.api_key_id,
+                            instance_id=current_instance,
+                            owner_epoch=session.durable_owner_epoch,
+                            lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(),
+                            latest_turn_state=session.downstream_turn_state,
+                            latest_response_id=None,
+                        )
+                    except Exception as exc:
+                        session.closed = True
+                        session.upstream_control.reconnect_requested = True
+                        session.upstream_control.retire_after_drain = True
+                        raise ProxyResponseError(
+                            502,
+                            openai_error(
+                                "bridge_continuity_persistence_failed",
+                                "HTTP responses session ownership could not be renewed; retry the request.",
+                            ),
+                        ) from exc
+                    if (
+                        owner_lookup is None
+                        or owner_lookup.owner_instance_id != current_instance
+                        or owner_lookup.owner_epoch != session.durable_owner_epoch
+                    ):
+                        session.closed = True
+                        session.upstream_control.reconnect_requested = True
+                        session.upstream_control.retire_after_drain = True
+                        raise ProxyResponseError(
+                            502,
+                            openai_error(
+                                "bridge_continuity_persistence_failed",
+                                "HTTP responses session ownership changed during cooldown; retry the request.",
+                            ),
+                        )
+            finally:
+                async with session.pending_lock:
+                    session.queued_request_count = max(0, session.queued_request_count - 1)
+            return True
+
+        async def operation_fenced_request_budget_terminal_event() -> str | None:
+            if not operation_fenced_cooldown_wait_enabled() or _service_time().monotonic() < request_deadline:
+                return None
+            await self._release_websocket_request_state_reservation(request_state)
+            request_state.api_key_reservation = None
+            if propagate_http_errors:
+                raise ProxyResponseError(
+                    503,
+                    openai_error(
+                        "upstream_request_timeout",
+                        "HTTP responses session bridge recovery exceeded the request budget.",
+                        error_type="server_error",
+                    ),
+                )
+            return format_sse_event(
+                cast(
+                    Mapping[str, JsonValue],
+                    response_failed_event(
+                        "stream_idle_timeout",
+                        "HTTP responses session bridge recovery exceeded the request budget",
+                        response_id=_websocket_downstream_response_id(request_state),
+                    ),
+                )
             )
 
         async def startup_continuity_cooldown_terminal_event() -> str | None:
@@ -3716,6 +3855,12 @@ async def startup_continuity_cooldown_terminal_event() -> str | None:
             )
 
         while True:
+            budget_terminal_event = await operation_fenced_request_budget_terminal_event()
+            if budget_terminal_event is not None:
+                yield budget_terminal_event
+                return
+            if await wait_through_operation_fenced_startup_cooldown():
+                continue
             startup_terminal_event = await startup_continuity_cooldown_terminal_event()
             if startup_terminal_event is not None:
                 yield startup_terminal_event

openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-08-14

openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md

@@ -0,0 +1,41 @@
+## Context
+
+Hard turn-state requests can omit `previous_response_id` while still carrying a
+real Codex turn-state continuity anchor. Their replay identity is protected by
+the durable operation ledger, but the startup cooldown guard runs before
+operation registration. It therefore classifies the request as continuity-bound
+without safe replay and returns 503 before the ledger can serialize recovery.
+
+The HTTP response already includes `Retry-After`, and an already-started SSE
+failure includes an SSE `retry:` directive. Production telemetry shows Codex
+Desktop retrying in milliseconds anyway, so another client hint does not address
+the observed failure mode.
+
+## Decision
+
+Treat a turn-state-only hard request as eligible to wait through cooldown only
+when all of the following hold:
+
+- recovery mode is `server_anchored_replay_once` or
+  `server_indefinite_recovery`;
+- the durable operation ledger is enabled;
+- the request has a real hard continuity anchor;
+- the bridge has both a durable session id and current owner epoch;
+- no response id or upstream response event has been observed; and
+- request budget remains.
+
+The wait is clamped to the smaller of cooldown remaining and request budget.
+It does not reserve a replay, mutate the operation journal, or send upstream.
+When the cooldown expires, normal submission performs the existing operation
+fingerprint lookup and atomic recovery claim. One-shot mode keeps its existing
+maximum of one recovery dispatch; indefinite mode retains its existing explicit
+opt-in semantics.
+
+## Explicit exclusions
+
+- No change to the default `fail_closed` mode.
+- No transparent replay without a durable session and owner fence.
+- No cross-account, file-pinned, image, soft-affinity, or eventful recovery.
+- No weakening of operation fingerprint, ownership, or replay-count checks.
+- No infinite retry added by this change; bounded one-shot mode is the
+  recommended deployment setting for this incident class.

openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md

@@ -0,0 +1,27 @@
+## Why
+
+When two eventless upstream attempts open the HTTP bridge retry circuit, Codex
+Desktop immediately retries the same hard turn-state request. The bridge
+currently returns a startup 503 before consulting the durable operation ledger.
+Codex does not honor the full retry-circuit delay and can exhaust its client
+retry budget during the cooldown, pausing the task even though the bridge and
+VPS remain healthy.
+
+## What Changes
+
+- In an explicitly enabled server recovery mode, hold a turn-state-only hard
+  continuation through the active retry-circuit cooldown before submission.
+- Require a live durable session id and owner epoch, zero response events, and
+  no response id before waiting.
+- Dispatch nothing while waiting. After cooldown, use the existing durable
+  operation ledger and one-shot/indefinite recovery policy to arbitrate whether
+  the request may be created, claimed, replayed, or failed closed.
+- Preserve the current immediate 503 for the default `fail_closed` mode,
+  in-memory fallback sessions, soft affinity, and eventful requests.
+- Emit a low-cardinality bridge event when the operation-fenced wait begins.
+
+## Impact
+
+- HTTP Responses bridge startup behavior during retry-circuit cooldown.
+- No database schema, public API, account routing, or default configuration
+  change.

openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md

@@ -0,0 +1,67 @@
+## ADDED Requirements
+
+### Requirement: Operation-fenced hard turns preserve client retry budget during cooldown
+
+A hard turn-state HTTP bridge request arriving during retry-circuit cooldown MUST remain pending until cooldown expires only if an explicit server recovery mode is enabled, the request has not observed a response id or response event, and the bridge has a live durable session and owner epoch. The proxy MUST NOT dispatch upstream while waiting. After the wait, the request MUST pass through the existing durable operation-ledger admission before any `response.create` is sent.
+
+#### Scenario: One-shot hard turn waits before durable arbitration
+
+- **GIVEN** `server_anchored_replay_once` is enabled
+- **AND** a turn-state-only hard continuation has a live durable owner
+- **AND** its retry circuit is cooling down before submission
+- **WHEN** the request reaches bridge startup
+- **THEN** the proxy waits for the bounded cooldown instead of returning 503
+- **AND** it sends no upstream request during the wait
+- **AND** normal durable operation admission runs after cooldown
+
+#### Scenario: Missing durable fence remains fail closed
+
+- **GIVEN** a turn-state-only hard continuation has no durable session or owner
+  epoch
+- **WHEN** its retry circuit is cooling down
+- **THEN** the proxy does not wait or dispatch upstream
+- **AND** it returns the existing cooldown failure with a retry hint
+
+#### Scenario: Operation ledger disabled remains fail closed
+
+- **GIVEN** ambiguous continuation recovery mode is enabled
+- **AND** a turn-state-only hard continuation has a live durable session and
+  owner epoch
+- **AND** the durable operation ledger is disabled
+- **WHEN** its retry circuit is cooling down before submission
+- **THEN** the proxy preserves the existing cooldown failure
+- **AND** it does not wait or dispatch upstream
+
+#### Scenario: Default mode remains fail closed
+
+- **GIVEN** ambiguous continuation recovery mode is `fail_closed`
+- **WHEN** any continuity-bound hard request arrives during cooldown
+- **THEN** the proxy preserves the existing immediate cooldown failure
+- **AND** it does not create or claim a durable recovery operation
+
+#### Scenario: Request budget expires while waiting
+
+- **GIVEN** an operation-fenced hard turn is allowed to wait through cooldown
+- **AND** its request budget expires before the cooldown does
+- **WHEN** the bounded wait reaches the request deadline
+- **THEN** the proxy releases the request reservation and returns a terminal
+  timeout
+- **AND** it does not submit `response.create` after the deadline
+
+#### Scenario: Cooldown waiter stays within the per-session queue limit
+
+- **GIVEN** an operation-fenced hard turn is eligible to wait through cooldown
+- **AND** the bridge session is already at its configured queue limit
+- **WHEN** the request reaches the cooldown wait point before submission
+- **THEN** the proxy rejects the request with the existing bridge queue full
+  error
+- **AND** it does not sleep or dispatch upstream
+
+#### Scenario: Durable ownership is renewed while the cooldown wait is pending
+
+- **GIVEN** an operation-fenced hard turn is waiting through startup cooldown
+- **AND** the cooldown exceeds one durable lease refresh cadence
+- **WHEN** the wait continues before submission
+- **THEN** the proxy renews and revalidates the durable owner lease before the
+  wait completes
+- **AND** it fails closed if durable ownership changes during the wait

openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md

@@ -0,0 +1,8 @@
+- [x] 1. Reproduce the production turn-state-only startup cooldown as a unit
+      regression that currently returns 503 before submission.
+- [x] 2. Hold only explicitly enabled, zero-event, durable operation-fenced hard
+      turns through the bounded cooldown.
+- [x] 3. Preserve fail-closed behavior when the durable session/owner proof is
+      absent and keep one-shot recovery bounded by the existing atomic claim.
+- [x] 4. Run focused tests, relevant bridge suites, Ruff, type/architecture
+      checks, whitespace checks, and strict OpenSpec validation.