diff --git a/app/core/clients/proxy_websocket.py b/app/core/clients/proxy_websocket.py index 06eb2d5fbb..8e85a3d07f 100644 --- a/app/core/clients/proxy_websocket.py +++ b/app/core/clients/proxy_websocket.py @@ -5,6 +5,8 @@ import logging import os import re +import threading +import weakref from dataclasses import dataclass from enum import StrEnum from typing import Any, Mapping, NoReturn, Protocol, Sequence, cast @@ -46,6 +48,7 @@ from app.core.openai.parsing import parse_error_payload from app.core.resilience.network_recovery import ( PROCESS_NETWORK_UNAVAILABLE_CODE, + correlate_websocket_egress_failure, process_network_error_code, rotate_shared_http_transport, ) @@ -97,6 +100,7 @@ class _UpstreamWebSocketPolicy: retry_routed_network_errors: bool enable_direct_ping_timeout: bool preserve_close_semantics: bool + correlate_no_close_failures: bool _RESPONSES_WEBSOCKET_POLICY = _UpstreamWebSocketPolicy( @@ -110,6 +114,7 @@ class _UpstreamWebSocketPolicy: retry_routed_network_errors=True, enable_direct_ping_timeout=False, preserve_close_semantics=False, + correlate_no_close_failures=True, ) _LIVE_SIDEBAND_WEBSOCKET_POLICY = _UpstreamWebSocketPolicy( operation="live websocket", @@ -122,6 +127,7 @@ class _UpstreamWebSocketPolicy: retry_routed_network_errors=False, enable_direct_ping_timeout=True, preserve_close_semantics=True, + correlate_no_close_failures=False, ) logger = logging.getLogger(__name__) @@ -149,6 +155,10 @@ class UpstreamWebSocketMessage: error_code: str | None = None +_websocket_receive_classification_lock = threading.Lock() +_websocket_receive_classification_tasks: weakref.WeakSet[asyncio.Future[Any]] = weakref.WeakSet() + + class UpstreamWebSocketTransportError(RuntimeError): """Credential-safe post-connect transport failure with stable classification.""" @@ -173,6 +183,60 @@ def _relay_receive_error_code(error_code: str) -> str | None: return error_code if error_code == PROCESS_NETWORK_UNAVAILABLE_CODE else None +def _discard_websocket_receive_classification_task(task: asyncio.Future[Any]) -> None: + with _websocket_receive_classification_lock: + _websocket_receive_classification_tasks.discard(task) + + +async def _correlate_no_close_receive_error( + error_code: str, + *, + enabled: bool, + account_id: str | None, + egress_key: str | None, +) -> str: + if not enabled: + return error_code + wait_for_correlation = error_code != PROCESS_NETWORK_UNAVAILABLE_CODE + if wait_for_correlation: + classification_owner = asyncio.current_task() + if classification_owner is not None: + with _websocket_receive_classification_lock: + _websocket_receive_classification_tasks.add(classification_owner) + classification_owner.add_done_callback(_discard_websocket_receive_classification_task) + try: + correlated = await correlate_websocket_egress_failure( + egress_key=egress_key, + account_id=account_id, + wait_for_correlation=wait_for_correlation, + ) + except asyncio.CancelledError: + raise + except Exception: + # Correlation is a health-classification guard, not a reason to lose + # the established stream-incomplete settlement path. + logger.warning("Responses websocket egress-failure correlation failed", exc_info=True) + return error_code + return PROCESS_NETWORK_UNAVAILABLE_CODE if correlated else error_code + + +async def await_pending_websocket_receive_classification( + receive_task: asyncio.Task[UpstreamWebSocketMessage] | None, +) -> bool: + """Let an observed no-close failure finish classification before settlement.""" + + if receive_task is None: + return False + if receive_task.done(): + return True + with _websocket_receive_classification_lock: + classification_pending = receive_task in _websocket_receive_classification_tasks + if not classification_pending: + return receive_task.done() + await asyncio.shield(receive_task) + return True + + async def _rotate_after_websocket_network_failure(error_code: str) -> None: if error_code != PROCESS_NETWORK_UNAVAILABLE_CODE: return @@ -219,10 +283,16 @@ def __init__( *, uses_proxy: bool = False, preserve_close_semantics: bool = False, + correlate_no_close_failures: bool = False, + account_id: str | None = None, + egress_key: str | None = None, ) -> None: self._connection = connection self._uses_proxy = uses_proxy self._preserve_close_semantics = preserve_close_semantics + self._correlate_no_close_failures = correlate_no_close_failures + self._account_id = account_id + self._egress_key = egress_key async def send_text(self, text: str) -> None: try: @@ -254,6 +324,13 @@ async def receive(self) -> UpstreamWebSocketMessage: ) error_code = _websocket_transport_error_code(exc, uses_proxy=self._uses_proxy) await _rotate_after_websocket_network_failure(error_code) + if exc.rcvd is None: + error_code = await _correlate_no_close_receive_error( + error_code, + enabled=self._correlate_no_close_failures, + account_id=self._account_id, + egress_key=self._egress_key, + ) # ConnectionClosedError describes an incomplete close handshake, # not generic transport provenance. Let Responses relay owners map # it to stream_incomplete while live relays preserve received closes. @@ -270,6 +347,12 @@ async def receive(self) -> UpstreamWebSocketMessage: except Exception as exc: error_code = _websocket_transport_error_code(exc, uses_proxy=self._uses_proxy) await _rotate_after_websocket_network_failure(error_code) + error_code = await _correlate_no_close_receive_error( + error_code, + enabled=self._correlate_no_close_failures, + account_id=self._account_id, + egress_key=self._egress_key, + ) return UpstreamWebSocketMessage( kind="error", error=codex_transport_error_message("websocket receive", None, exc), @@ -310,6 +393,9 @@ def __init__( owns_codex_client: bool = False, endpoint_id: str | None = None, response_headers: Mapping[str, str] | None = None, + correlate_no_close_failures: bool = False, + account_id: str | None = None, + egress_key: str | None = None, ) -> None: self._websocket = websocket self._context = context @@ -317,6 +403,10 @@ def __init__( self._owns_codex_client = owns_codex_client self._endpoint_id = endpoint_id self._response_headers = _normalize_response_headers(response_headers) + self._correlate_no_close_failures = correlate_no_close_failures + self._account_id = account_id + self._egress_key = egress_key + self._received_peer_close_frame = False async def send_text(self, text: str) -> None: try: @@ -340,12 +430,41 @@ async def receive(self) -> UpstreamWebSocketMessage: except Exception as exc: error_code = _websocket_transport_error_code(exc, uses_proxy=True) await _rotate_after_websocket_network_failure(error_code) + error_code = await _correlate_no_close_receive_error( + error_code, + enabled=self._correlate_no_close_failures, + account_id=self._account_id, + egress_key=self._egress_key, + ) return UpstreamWebSocketMessage( kind="error", error=codex_transport_error_message("websocket receive", self._endpoint_id, exc), error_code=_relay_receive_error_code(error_code), ) - if msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSING, aiohttp.WSMsgType.CLOSED): + if msg.type == aiohttp.WSMsgType.CLOSE: + self._received_peer_close_frame = True + return UpstreamWebSocketMessage( + kind="close", + close_code=_aiohttp_ws_close_code(self._websocket, msg), + close_reason=_aiohttp_ws_close_reason(msg), + ) + if ( + msg.type == aiohttp.WSMsgType.CLOSED + and not self._received_peer_close_frame + and self._correlate_no_close_failures + ): + error_code = await _correlate_no_close_receive_error( + "upstream_unavailable", + enabled=self._correlate_no_close_failures, + account_id=self._account_id, + egress_key=self._egress_key, + ) + return UpstreamWebSocketMessage( + kind="error", + error="Upstream websocket closed without a peer close frame", + error_code=_relay_receive_error_code(error_code), + ) + if msg.type in (aiohttp.WSMsgType.CLOSING, aiohttp.WSMsgType.CLOSED): return UpstreamWebSocketMessage( kind="close", close_code=_aiohttp_ws_close_code(self._websocket, msg), @@ -359,6 +478,12 @@ async def receive(self) -> UpstreamWebSocketMessage: else "upstream_unavailable" ) await _rotate_after_websocket_network_failure(error_code) + error_code = await _correlate_no_close_receive_error( + error_code, + enabled=self._correlate_no_close_failures, + account_id=self._account_id, + egress_key=self._egress_key, + ) return UpstreamWebSocketMessage( kind="error", error=( @@ -682,6 +807,44 @@ def _responses_websocket_url(base_url: str) -> str: return urlunparse(parsed._replace(scheme=scheme)) +def _websocket_egress_key( + url: str, + *, + route_endpoint_id: str | None = None, + proxy_url: str | None = None, +) -> str | None: + if route_endpoint_id: + return f"routed_proxy:{route_endpoint_id}" + + candidate_url = proxy_url or url + parsed = urlparse(candidate_url) + scheme = parsed.scheme.lower() + hostname = parsed.hostname + if not scheme or not hostname: + return None + try: + port = parsed.port + except ValueError: + return None + if port is None: + port = { + "http": 80, + "https": 443, + "socks": 1080, + "socks4": 1080, + "socks4a": 1080, + "socks5": 1080, + "socks5h": 1080, + "ws": 80, + "wss": 443, + }.get(scheme) + if port is None: + return None + authority_host = f"[{hostname}]" if ":" in hostname else hostname + prefix = "environment_proxy" if proxy_url is not None else "direct" + return f"{prefix}:{scheme}://{authority_host}:{port}" + + async def _connect_upstream_websocket( headers: dict[str, str], access_token: str, @@ -785,6 +948,12 @@ async def _connect_upstream_websocket( owns_codex_client=owns_codex_client, endpoint_id=endpoint_id, response_headers=_codex_websocket_response_headers(websocket, context), + correlate_no_close_failures=policy.correlate_no_close_failures, + account_id=account_id, + egress_key=_websocket_egress_key( + url, + route_endpoint_id=endpoint_id, + ), ), url=url, headers=upstream_headers, @@ -882,6 +1051,12 @@ async def _connect_upstream_websocket( response, uses_proxy=proxy_url is not None, preserve_close_semantics=policy.preserve_close_semantics, + correlate_no_close_failures=policy.correlate_no_close_failures, + account_id=account_id, + egress_key=_websocket_egress_key( + url, + proxy_url=proxy_url, + ), ), url=url, headers=upstream_headers, diff --git a/app/core/resilience/network_recovery.py b/app/core/resilience/network_recovery.py index 51bb9f3735..ea43fe905e 100644 --- a/app/core/resilience/network_recovery.py +++ b/app/core/resilience/network_recovery.py @@ -4,7 +4,9 @@ import errno import logging import socket +import threading import time +from collections import deque from collections.abc import Iterator from dataclasses import dataclass from typing import Literal @@ -37,10 +39,156 @@ # Keep that lifecycle cleanup independently bounded in case client construction # or lock acquisition stalls. _CONCRETE_FAILED_GENERATION_ROTATION_TIMEOUT_SECONDS = 5.0 +_WEBSOCKET_EGRESS_FAILURE_CORRELATION_WINDOW_SECONDS = 1.0 +_WEBSOCKET_EGRESS_FAILURE_MAX_OBSERVATIONS = 1024 NetworkRecoveryDecision = Literal["not_applicable", "retry", "exhausted"] +@dataclass(slots=True) +class _WebSocketEgressFailureObservation: + egress_key: str + account_id: str + observed_at: float + loop: asyncio.AbstractEventLoop + waiter: asyncio.Future[bool] | None + correlated: bool = False + + +def _resolve_websocket_egress_failure_waiter( + waiter: asyncio.Future[bool], + result: bool, +) -> None: + if not waiter.done(): + waiter.set_result(result) + + +class WebSocketEgressFailureCorrelator: + """Bound ambiguous no-close failures until cross-account evidence arrives.""" + + def __init__( + self, + *, + window_seconds: float = _WEBSOCKET_EGRESS_FAILURE_CORRELATION_WINDOW_SECONDS, + max_observations: int = _WEBSOCKET_EGRESS_FAILURE_MAX_OBSERVATIONS, + ) -> None: + if window_seconds <= 0: + raise ValueError("window_seconds must be positive") + if max_observations <= 0: + raise ValueError("max_observations must be positive") + self._window_seconds = window_seconds + self._max_observations = max_observations + self._lock = threading.Lock() + self._observations: deque[_WebSocketEgressFailureObservation] = deque() + + async def observe( + self, + *, + egress_key: str | None, + account_id: str | None, + wait_for_correlation: bool = True, + ) -> bool: + """Record an egress failure and optionally await cross-account evidence.""" + + if not egress_key or not account_id: + return False + + loop = asyncio.get_running_loop() + waiter: asyncio.Future[bool] | None = loop.create_future() if wait_for_correlation else None + with self._lock: + observed_at = time.monotonic() + self._expire_observations_locked(observed_at) + while len(self._observations) >= self._max_observations: + # Capacity pressure removes correlation evidence, but it must + # not let an evicted candidate reach health settlement before + # its own bounded judgment window ends. + self._observations.popleft().waiter = None + observation = _WebSocketEgressFailureObservation( + egress_key=egress_key, + account_id=account_id, + observed_at=observed_at, + loop=loop, + waiter=waiter, + ) + self._observations.append(observation) + matching = [ + candidate + for candidate in self._observations + if candidate.egress_key == egress_key and observed_at - candidate.observed_at <= self._window_seconds + ] + correlated = len({candidate.account_id for candidate in matching}) >= 2 + if correlated: + for candidate in matching: + candidate.correlated = True + self._notify_observation_locked(candidate, result=True) + + if correlated: + return True + if not wait_for_correlation: + return False + + assert waiter is not None + try: + return await asyncio.wait_for( + asyncio.shield(waiter), + timeout=self._window_seconds, + ) + except TimeoutError: + with self._lock: + correlated = observation.correlated + if observation.waiter is waiter: + observation.waiter = None + waiter.cancel() + return correlated + except asyncio.CancelledError: + with self._lock: + if observation.waiter is waiter: + observation.waiter = None + waiter.cancel() + raise + + def _expire_observations_locked(self, now: float) -> None: + while self._observations and now - self._observations[0].observed_at > self._window_seconds: + self._notify_observation_locked(self._observations.popleft(), result=False) + + @staticmethod + def _notify_observation_locked( + observation: _WebSocketEgressFailureObservation, + *, + result: bool, + ) -> None: + waiter = observation.waiter + if waiter is None: + return + observation.waiter = None + try: + observation.loop.call_soon_threadsafe( + _resolve_websocket_egress_failure_waiter, + waiter, + result, + ) + except RuntimeError: + # A closed event loop owns no remaining health settlement. The + # bounded observation itself still expires normally. + return + + +_websocket_egress_failure_correlator = WebSocketEgressFailureCorrelator() + + +async def correlate_websocket_egress_failure( + *, + egress_key: str | None, + account_id: str | None, + wait_for_correlation: bool = True, +) -> bool: + return await _websocket_egress_failure_correlator.observe( + egress_key=egress_key, + account_id=account_id, + wait_for_correlation=wait_for_correlation, + ) + + def _exception_chain(exc: BaseException) -> Iterator[BaseException]: seen: set[int] = set() pending: list[BaseException] = [exc] diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index ca98a7fa9f..257cd78e1d 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -188,6 +188,7 @@ logger = logging.getLogger("app.modules.proxy.service") _HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS = 5.0 +_HTTP_BRIDGE_DURABLE_ANCHOR_QUARANTINE_TIMEOUT_SECONDS = 5.0 _HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 240.0 _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL = "missing_response_created_timeout" T = TypeVar("T") @@ -1621,6 +1622,148 @@ async def _persist_http_bridge_previous_response_alias( return registered +async def _quarantine_http_bridge_durable_anchor( + service: _HTTPBridgeServiceProtocol, + session: _HTTPBridgeSession, + *, + request_state: _WebSocketRequestState, +) -> bool: + expected_response_id = request_state.previous_response_id + if not request_state.proxy_injected_previous_response_id or expected_response_id is None: + return False + return await _quarantine_http_bridge_durable_anchor_id( + service, + session, + expected_response_id=expected_response_id, + ) + + +async def _quarantine_http_bridge_durable_anchor_id( + service: _HTTPBridgeServiceProtocol, + session: _HTTPBridgeSession, + *, + expected_response_id: str, +) -> bool: + session_id = session.durable_session_id + owner_epoch = session.durable_owner_epoch + if session_id is None or owner_epoch is None: + outcome = "missing_durable_identity" + else: + instance_id = _service_get_settings().http_responses_session_bridge_instance_id + try: + lookup = await asyncio.wait_for( + service._durable_bridge.clear_latest_response_anchor_if_current( + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + expected_response_id=expected_response_id, + ), + timeout=_HTTP_BRIDGE_DURABLE_ANCHOR_QUARANTINE_TIMEOUT_SECONDS, + ) + except TimeoutError: + logger.warning( + "Timed out quarantining durable HTTP bridge latest-response anchor timeout_seconds=%.1f", + _HTTP_BRIDGE_DURABLE_ANCHOR_QUARANTINE_TIMEOUT_SECONDS, + ) + outcome = "write_timeout" + except Exception: + logger.warning("Failed to quarantine durable HTTP bridge latest-response anchor", exc_info=True) + outcome = "write_error" + else: + if lookup is None: + outcome = "row_missing" + elif lookup.owner_instance_id != instance_id or lookup.owner_epoch != owner_epoch: + outcome = "owner_fenced" + elif lookup.latest_response_id is None: + outcome = "cleared" + elif lookup.latest_response_id != expected_response_id: + outcome = "newer_anchor_preserved" + else: + outcome = "conditional_clear_missed" + _log_http_bridge_event( + "durable_anchor_quarantine", + session.key, + account_id=session.account.id, + model=session.request_model, + detail=f"outcome={outcome}", + cache_key_family=session.key.affinity_kind, + model_class=_extract_model_class(session.request_model) if session.request_model else None, + owner_check_applied=True, + ) + return outcome == "cleared" + + +async def _quarantine_http_bridge_disconnected_socket_anchor( + service: _HTTPBridgeServiceProtocol, + session: _HTTPBridgeSession, + *, + expected_response_id: str, + lifecycle_lock_held: bool = False, +) -> bool: + """Fence one dead socket anchor and clear only matching local provenance.""" + + if not lifecycle_lock_held: + async with session.lifecycle_lock: + return await _quarantine_http_bridge_disconnected_socket_anchor( + service, + session, + expected_response_id=expected_response_id, + lifecycle_lock_held=True, + ) + quarantined = await _quarantine_http_bridge_durable_anchor_id( + service, + session, + expected_response_id=expected_response_id, + ) + if quarantined and session.last_completed_response_id == expected_response_id: + session.last_completed_response_id = None + session.last_completed_response_store = None + session.last_pending_tool_calls = {} + return quarantined + + +def _select_http_bridge_disconnect_anchor_quarantine_response_id( + pending_requests: Sequence[_WebSocketRequestState], + *, + latest_response_id: str | None, + latest_response_store: bool | None, +) -> str | None: + """Select one sent connection-local anchor id while the pending lock is held.""" + + eligible = [ + request_state + for request_state in pending_requests + if request_state.transport == "http" + and not request_state.skip_request_log + and not request_state.draining_until_terminal + and request_state.response_create_sent_at is not None + and request_state.response_store is False + and request_state.proxy_injected_previous_response_id + and request_state.previous_response_id is not None + ] + gate_owners = [ + request_state + for request_state in eligible + if request_state.response_create_gate_acquired and request_state.awaiting_response_created + ] + if len(gate_owners) == 1: + return gate_owners[0].previous_response_id + if latest_response_id is not None: + for request_state in eligible: + if request_state.previous_response_id == latest_response_id: + return latest_response_id + candidates_by_anchor: dict[str, _WebSocketRequestState] = {} + for request_state in eligible: + previous_response_id = request_state.previous_response_id + if previous_response_id is not None: + candidates_by_anchor.setdefault(previous_response_id, request_state) + if len(candidates_by_anchor) == 1: + return next(iter(candidates_by_anchor)) + if not candidates_by_anchor and latest_response_id is not None and latest_response_store is False: + return latest_response_id + return None + + def _evict_fenced_out_http_bridge_session_locked( service: _HTTPBridgeServiceProtocol, session: _HTTPBridgeSession, @@ -1997,6 +2140,19 @@ def _http_bridge_continuity_lost_error_envelope() -> OpenAIErrorEnvelope: return previous_response_stream_incomplete_error() +def _http_bridge_full_resend_required_error() -> ProxyResponseError: + payload = openai_error( + "continuity_requires_full_resend", + ( + "HTTP bridge continuity cannot resume incrementally. " + "Resend the complete conversation context in input or create a new session." + ), + error_type="invalid_request_error", + ) + payload["error"]["param"] = "input" + return ProxyResponseError(400, payload) + + def _http_bridge_owner_lookup_unavailable_error_envelope() -> OpenAIErrorEnvelope: return openai_error( "upstream_unavailable", diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index d8dfeb0c51..f8a9921436 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -84,6 +84,7 @@ _http_bridge_continuity_lost_error_envelope, _http_bridge_endpoint_matches_current_instance, _http_bridge_eviction_priority, + _http_bridge_full_resend_required_error, _http_bridge_has_durable_recovery_anchor, _http_bridge_incompatible_model_fork_key, _http_bridge_key_strength, @@ -352,6 +353,7 @@ async def _get_or_create_http_bridge_session( forwarded_affinity_key: str | None = None, allow_previous_response_recovery_rebind: bool = False, allow_bootstrap_owner_rebind: bool = False, + allow_fresh_session_creation: bool = True, durable_lookup: DurableBridgeLookup | None = None, request_stage: str = "first_turn", preferred_account_id: str | None = None, @@ -384,6 +386,7 @@ async def _get_or_create_http_bridge_session( forwarded_affinity_key: str | None = None, allow_previous_response_recovery_rebind: bool = False, allow_bootstrap_owner_rebind: bool = False, + allow_fresh_session_creation: bool = True, durable_lookup: DurableBridgeLookup | None = None, request_stage: str = "first_turn", preferred_account_id: str | None = None, @@ -415,6 +418,7 @@ async def _get_or_create_http_bridge_session( forwarded_affinity_key: str | None = None, allow_previous_response_recovery_rebind: bool = False, allow_bootstrap_owner_rebind: bool = False, + allow_fresh_session_creation: bool = True, durable_lookup: DurableBridgeLookup | None = None, request_stage: str = "first_turn", preferred_account_id: str | None = None, @@ -493,6 +497,16 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: ("local account-neutral recovery", session.account.id), ) + def fresh_session_creation_rejected_error() -> ProxyResponseError: + _record_continuity_fail_closed( + surface="http_bridge", + reason="fresh_socket_anchor_requires_full_resend", + previous_response_id=previous_response_id, + session_id=incoming_turn_state or incoming_session_key, + upstream_error_code="durable_anchor_from_prior_socket", + ) + return _http_bridge_full_resend_required_error() + while True: account_neutral_recovery = is_http_bridge_account_neutral_replay( kind=key.affinity_kind, @@ -1248,6 +1262,20 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: model_class=_extract_model_class(request_model) if request_model else None, owner_check_applied=owner_check_required, ) + if ( + continuity_error is None + and session_to_return_after_close is None + and owner_forward is None + and not allow_fresh_session_creation + ): + continuity_error = fresh_session_creation_rejected_error() + elif ( + session_to_return_after_close is None + and inflight_future is None + and owner_forward is None + and not allow_fresh_session_creation + ): + continuity_error = fresh_session_creation_rejected_error() elif inflight_future is None: while ( len(self._http_bridge_sessions) + len(self._http_bridge_inflight_sessions) >= max_sessions @@ -2004,7 +2032,8 @@ async def _reconnect_http_bridge_session( kind=session.key.affinity_kind, key=session.key.affinity_key, ) - require_same_account = require_same_account or account_neutral_recovery + account_bound_owner_id = request_state.account_bound_owner_id + require_same_account = require_same_account or account_neutral_recovery or account_bound_owner_id is not None old_account_id = session.account.id old_upstream = session.upstream old_reader = session.upstream_reader if restart_reader else None @@ -2029,9 +2058,12 @@ async def _reconnect_http_bridge_session( forced_refresh_account_id = request_state.force_refresh_account_id excluded_account_ids: set[str] = set(request_state.excluded_account_ids) requested_preferred_account_id = ( - request_state.preferred_account_id if require_preferred_account or account_neutral_recovery else None + request_state.preferred_account_id + if require_preferred_account or account_neutral_recovery or account_bound_owner_id is not None + else None ) required_preferred_account_id = resolve_required_account_id( + ("account-bound request", account_bound_owner_id), ("requested reconnect owner", requested_preferred_account_id), ( "account-neutral recovery", @@ -2039,7 +2071,9 @@ async def _reconnect_http_bridge_session( ), ) close_skips_account = session.last_upstream_close_code in _UPSTREAM_CLOSE_CODES_SKIP_SAME_ACCOUNT_RETRY - hard_close_account_bound = session.key.strength == "hard" and (close_skips_account or require_same_account) + hard_close_account_bound = account_bound_owner_id is not None or ( + session.key.strength == "hard" and (close_skips_account or require_same_account) + ) skip_same_account = ( session.key.strength != "hard" and close_skips_account and required_preferred_account_id is None ) @@ -2295,6 +2329,7 @@ async def abandon_selected_account_retry(selected_account: Any) -> None: await self._unregister_http_bridge_turn_states(session) await self._unregister_http_bridge_previous_response_ids(session) session.last_completed_response_id = None + session.last_completed_response_store = None session.last_completed_input_count = 0 session.last_completed_input_prefix_fingerprint = None session.last_pending_tool_calls.clear() @@ -2313,6 +2348,7 @@ async def abandon_selected_account_retry(selected_account: Any) -> None: replaced_account_lease = session.account_lease session.account_lease = selected_account_lease session.account, session.headers, session.upstream = account, connect_headers, upstream + session.last_completed_response_store = None session.catalog_omission_quota_admission = selection.catalog_omission_quota_admission session.upstream_control = _WebSocketUpstreamControl() session.closed = False diff --git a/app/modules/proxy/_service/http_bridge/owner_forwarding.py b/app/modules/proxy/_service/http_bridge/owner_forwarding.py index 51cdbda989..13fe3ac98a 100644 --- a/app/modules/proxy/_service/http_bridge/owner_forwarding.py +++ b/app/modules/proxy/_service/http_bridge/owner_forwarding.py @@ -345,6 +345,7 @@ async def _forward_http_bridge_request_to_owner( proxy_api_authorization: str | None, file_owner_account_id: str | None = None, client_ip: str | None = None, + proxy_injected_previous_response_id: bool = False, ) -> AsyncIterator[str]: current_instance, _ = _normalized_http_bridge_instance_ring(_service_get_settings()) incoming_turn_state = _sticky_key_from_turn_state_header(headers) @@ -373,6 +374,7 @@ async def _forward_http_bridge_request_to_owner( and payload.previous_response_id is None ) ), + proxy_injected_previous_response_id=proxy_injected_previous_response_id, original_affinity_kind=owner_forward.key.affinity_kind, original_affinity_key=owner_forward.key.affinity_key, file_owner_account_id=file_owner_account_id, diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 0319e918ea..1f6e0b179d 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -70,16 +70,19 @@ from app.modules.proxy._service.http_bridge.helpers import ( _await_task_deferring_cancellation, _build_http_bridge_prewarm_text, + _http_bridge_full_resend_required_error, _http_bridge_key_strength, _http_bridge_precreated_retry_failure_error, _http_bridge_prewarm_enabled, _http_bridge_request_budget_seconds, _http_bridge_request_counts_against_queue, _log_http_bridge_event, + _quarantine_http_bridge_disconnected_socket_anchor, _record_continuity_fail_closed, _record_http_bridge_prewarm_outcome, _register_http_bridge_turn_state_aliases_locked, _release_http_bridge_unanchored_handoff, + _select_http_bridge_disconnect_anchor_quarantine_response_id, ) from app.modules.proxy._service.http_bridge.service_stubs import ( _call_with_supported_optional_kwargs, @@ -272,6 +275,41 @@ def _request_kind_from_headers(headers: Mapping[str, str] | None) -> str: return "normal" +async def _inline_http_bridge_image_text( + text_data: str, + *, + image_fetch_session: ImageFetchSession, + connect_timeout: float, +) -> str: + if "input_image" not in text_data: + return text_data + try: + payload_dict: dict[str, JsonValue] = json.loads(text_data) + except (json.JSONDecodeError, TypeError): + return text_data + inlined = await _service_inline_input_image_urls()( + payload_dict, + image_fetch_session, + connect_timeout, + ) + inlined = await _inline_top_level_input_image_urls(inlined, image_fetch_session, connect_timeout) + remaining_external = _count_external_image_urls(inlined) + if remaining_external > 0: + raise ProxyResponseError( + 400, + openai_error( + "image_download_failed", + ( + f"Failed to download {remaining_external} external image(s). " + "The upstream API only accepts inline data: URLs. " + "Send images as base64 data URLs (data:image/png;base64,...) " + "or ensure the image URLs are publicly accessible." + ), + ), + ) + return json.dumps(inlined, ensure_ascii=True, separators=(",", ":")) + + class _HTTPBridgeRequestSubmitMixin: def _prepare_http_bridge_request( self: Any, @@ -382,6 +420,7 @@ def _prepare_response_bridge_request_state( api_key=api_key, request_usage_budget=estimate_api_key_request_usage(payload), previous_response_id=payload.previous_response_id, + response_store=payload.store, session_id=_normalize_session_id(session_id), input_item_count=input_item_count, input_full_fingerprint=input_full_fingerprint, @@ -474,41 +513,33 @@ async def _inline_http_bridge_image_urls( settings = _service_get_settings() if not settings.image_inline_fetch_enabled: return text_data - # Quick string-level pre-check: skip the parse/fetch cycle when the - # payload contains no ``input_image`` items with an ``http`` URL. - if "input_image" not in text_data: - return text_data - try: - payload_dict: dict[str, JsonValue] = json.loads(text_data) - except (json.JSONDecodeError, TypeError): + fresh_text = request_state.fresh_upstream_request_text + if "input_image" not in text_data and (fresh_text is None or "input_image" not in fresh_text): return text_data connect_timeout = getattr(settings, "upstream_connect_timeout_seconds", 5.0) async with _service_lease_http_session()() as http_session: image_fetch_session = _service_as_image_fetch_session()(http_session) - inlined = await _service_inline_input_image_urls()( - payload_dict, - image_fetch_session, - connect_timeout, + updated_text = await _inline_http_bridge_image_text( + text_data, + image_fetch_session=image_fetch_session, + connect_timeout=connect_timeout, ) - inlined = await _inline_top_level_input_image_urls(inlined, image_fetch_session, connect_timeout) - # After inlining, check if any external URLs survived (i.e. fetch - # failed). The upstream WS only accepts data: URLs so sending an - # external URL would just cause a silent hang. - remaining_external = _count_external_image_urls(inlined) - if remaining_external > 0: - raise ProxyResponseError( - 400, - openai_error( - "image_download_failed", - ( - f"Failed to download {remaining_external} external image(s). " - "The upstream API only accepts inline data: URLs. " - "Send images as base64 data URLs (data:image/png;base64,...) " - "or ensure the image URLs are publicly accessible." - ), - ), + if fresh_text is None: + updated_fresh_text = None + elif fresh_text == text_data: + updated_fresh_text = updated_text + else: + updated_fresh_text = await _inline_http_bridge_image_text( + fresh_text, + image_fetch_session=image_fetch_session, + connect_timeout=connect_timeout, + ) + if updated_fresh_text is not None and updated_fresh_text != fresh_text: + request_state.fresh_upstream_request_text = updated_fresh_text + _enforce_http_bridge_response_create_text_size( + request_state, + updated_fresh_text, ) - updated_text = json.dumps(inlined, ensure_ascii=True, separators=(",", ":")) if updated_text == text_data: return text_data request_state.request_text = updated_text @@ -787,6 +818,57 @@ async def _submit_http_bridge_request_with_handoff( 502, openai_error("upstream_unavailable", "HTTP responses session bridge is closed"), ) + if request_state.proxy_injected_previous_response_id and ( + request_state.previous_response_id is None + or session.last_completed_response_id != request_state.previous_response_id + or session.last_completed_response_store is not False + ): + stale_response_id = request_state.previous_response_id + if ( + request_state.fresh_upstream_request_is_retry_safe + and request_state.fresh_upstream_request_text is not None + ): + text_data = self._http_bridge_text_with_account_installation_id( + session, + request_state, + request_state.fresh_upstream_request_text, + ) + request_state.previous_response_id = None + request_state.proxy_injected_previous_response_id = False + request_state.request_text = text_data + _log_http_bridge_event( + "proxy_anchor_revalidated_before_send", + session.key, + account_id=session.account.id, + model=session.request_model, + detail="outcome=fresh_full_resend", + cache_key_family=session.key.affinity_kind, + model_class=( + _extract_model_class(session.request_model) if session.request_model else None + ), + owner_check_applied=True, + ) + else: + _record_continuity_fail_closed( + surface="http_bridge", + reason="proxy_injected_anchor_socket_changed_before_send", + previous_response_id=stale_response_id, + session_id=request_state.session_id, + upstream_error_code="durable_anchor_from_prior_socket", + ) + _log_http_bridge_event( + "proxy_anchor_revalidated_before_send", + session.key, + account_id=session.account.id, + model=session.request_model, + detail="outcome=continuity_failed_closed", + cache_key_family=session.key.affinity_kind, + model_class=( + _extract_model_class(session.request_model) if session.request_model else None + ), + owner_check_applied=True, + ) + raise _http_bridge_full_resend_required_error() recovery_receipt: DurableBridgeAliasRegistrationReceipt | None = None upstream_send_started = False try: @@ -1124,9 +1206,12 @@ async def _maybe_prewarm_http_bridge_session( session, request_state=request_state, restart_reader=True, - require_same_account=is_http_bridge_account_neutral_replay( - kind=session.key.affinity_kind, - key=session.key.affinity_key, + require_same_account=( + request_state.account_bound_owner_id is not None + or is_http_bridge_account_neutral_replay( + kind=session.key.affinity_kind, + key=session.key.affinity_key, + ) ), ) except Exception: @@ -1377,32 +1462,61 @@ async def _detach_http_bridge_request( request_state: _WebSocketRequestState, ) -> bool: detached = False - async with session.pending_lock: - if request_state in session.pending_requests and not request_state.draining_until_terminal: - request_state.draining_until_terminal = True - request_state.downstream_visible = False - session.queued_request_count = max(0, session.queued_request_count - 1) - session.upstream_control.reconnect_requested = True - session.upstream_control.retire_after_drain = True - detached = True - request_state.event_queue = None - # event_queue is nulled unconditionally because by the time - # _detach is called from the finally block in - # _stream_http_bridge_session_events, the terminal event has - # already been delivered via _pop_terminal_websocket_request_state. - # A late-arriving event on a nulled queue is a no-op. + disconnect_quarantine_response_id: str | None = None + async with session.lifecycle_lock: + async with session.pending_lock: + if request_state in session.pending_requests and not request_state.draining_until_terminal: + disconnect_quarantine_response_id = _select_http_bridge_disconnect_anchor_quarantine_response_id( + session.pending_requests, + latest_response_id=session.last_completed_response_id, + latest_response_store=session.last_completed_response_store, + ) + request_state.draining_until_terminal = True + request_state.downstream_visible = False + session.queued_request_count = max(0, session.queued_request_count - 1) + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + detached = True + request_state.event_queue = None + # event_queue is nulled unconditionally because by the time + # _detach is called from the finally block in + # _stream_http_bridge_session_events, the terminal event has + # already been delivered via _pop_terminal_websocket_request_state. + # A late-arriving event on a nulled queue is a no-op. + if disconnect_quarantine_response_id is not None: + await _quarantine_http_bridge_disconnected_socket_anchor( + self, + session, + expected_response_id=disconnect_quarantine_response_id, + lifecycle_lock_held=True, + ) + if detached: + await self._retire_http_bridge_after_drain_if_ready( + session, + lifecycle_lock_held=True, + ) await _release_websocket_response_create_gate(request_state, session.response_create_gate) if not detached: return False self._cancel_request_state_api_key_reservation_heartbeat(request_state) await self._release_websocket_request_state_reservation(request_state) request_state.api_key_reservation = None - await self._retire_http_bridge_after_drain_if_ready(session) return True - async def _retire_http_bridge_after_drain_if_ready(self: Any, session: "_HTTPBridgeSession") -> bool: + async def _retire_http_bridge_after_drain_if_ready( + self: Any, + session: "_HTTPBridgeSession", + *, + lifecycle_lock_held: bool = False, + ) -> bool: if not (session.upstream_control.reconnect_requested and session.upstream_control.retire_after_drain): return False + if not lifecycle_lock_held: + async with session.lifecycle_lock: + return await self._retire_http_bridge_after_drain_if_ready( + session, + lifecycle_lock_held=True, + ) async with session.pending_lock: has_visible_pending = any( _http_bridge_request_counts_against_queue(request_state) for request_state in session.pending_requests @@ -1457,9 +1571,13 @@ async def _retry_http_bridge_request_on_fresh_upstream( send_request: bool = True, require_same_account: bool = False, ) -> bool: - require_same_account = require_same_account or is_http_bridge_account_neutral_replay( - kind=session.key.affinity_kind, - key=session.key.affinity_key, + require_same_account = ( + require_same_account + or request_state.account_bound_owner_id is not None + or is_http_bridge_account_neutral_replay( + kind=session.key.affinity_kind, + key=session.key.affinity_key, + ) ) retry_text_data = text_data using_fresh_replay = False @@ -1556,6 +1674,9 @@ async def _retry_http_bridge_precreated_request( if len(retryable_requests) != 1: return False request_state = retryable_requests[0] + hard_owner_bound = hard_owner_bound or request_state.account_bound_owner_id is not None + if request_state.account_bound_owner_id is not None: + request_state.preferred_account_id = request_state.account_bound_owner_id if request_state.previous_response_id is not None and not ( request_state.proxy_injected_previous_response_id and request_state.fresh_upstream_request_is_retry_safe @@ -1732,6 +1853,9 @@ async def _retry_http_bridge_precreated_auth_request( error_message: str | None, ) -> Literal["not_replayable", "retried", "failed"]: permanent_failure_code = _websocket_auth_failure_permanent_code(error_message) + account_bound_owner_id = request_state.account_bound_owner_id + if account_bound_owner_id is not None and session.account.id != account_bound_owner_id: + return "not_replayable" request_text = _prepare_websocket_request_state_for_auth_replay(request_state) if request_text is None: await self._load_balancer.mark_permanent_failure(session.account, permanent_failure_code) @@ -1756,7 +1880,7 @@ async def _retry_http_bridge_precreated_auth_request( request_state.force_refresh_account_id = None request_state.preferred_account_id = None request_state.excluded_account_ids.add(session.account.id) - if is_http_bridge_account_neutral_replay( + if account_bound_owner_id is not None or is_http_bridge_account_neutral_replay( kind=session.key.affinity_kind, key=session.key.affinity_key, ): @@ -1781,9 +1905,12 @@ async def _retry_http_bridge_precreated_auth_request( await self._reconnect_http_bridge_session( session, request_state=request_state, - require_same_account=is_http_bridge_account_neutral_replay( - kind=session.key.affinity_kind, - key=session.key.affinity_key, + require_same_account=( + account_bound_owner_id is not None + or is_http_bridge_account_neutral_replay( + kind=session.key.affinity_kind, + key=session.key.affinity_key, + ) ), ) request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text) @@ -1831,6 +1958,8 @@ async def _retry_http_bridge_security_work_request( return False if request_state.file_required_preferred_account: return False + if request_state.account_bound_owner_id is not None: + return False if not _websocket_request_can_replay_before_visible_output(request_state): return False diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 1fce2706ea..ad1e236b48 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -67,6 +67,7 @@ from app.modules.proxy._service.http_bridge.helpers import ( _effective_http_bridge_idle_ttl_seconds, _http_bridge_durable_lookup_allows_turn_state_takeover, + _http_bridge_full_resend_required_error, _http_bridge_is_context_overflow_error, _http_bridge_is_previous_response_owner_unavailable, _http_bridge_models_compatible, @@ -209,9 +210,11 @@ ) from app.modules.proxy.replay_safety import ( project_responses_input_for_account_neutral_fresh_replay, + responses_input_has_self_contained_tool_continuation_suffix, responses_input_suffix_matches_pending_tool_calls, responses_input_suffix_retains_prior_output, responses_payload_is_account_neutral_fresh_replay, + responses_payload_is_same_account_compaction_recovery, ) logger = logging.getLogger("app.modules.proxy.service") @@ -220,6 +223,24 @@ _RESPONSE_CREATE_GATE_RETRY_SLEEP_SECONDS = 10.0 +def _preserve_proxy_anchor_recovery_state( + source: _WebSocketRequestState, + target: _WebSocketRequestState, +) -> None: + """Preserve request ownership and exact-anchor fallback across re-preparation.""" + + target.account_bound_owner_id = source.account_bound_owner_id + if ( + not source.proxy_injected_previous_response_id + or source.previous_response_id is None + or target.previous_response_id != source.previous_response_id + ): + return + target.proxy_injected_previous_response_id = True + target.fresh_upstream_request_text = source.fresh_upstream_request_text + target.fresh_upstream_request_is_retry_safe = source.fresh_upstream_request_is_retry_safe + + def _http_bridge_payload_is_account_neutral_fresh_replay(payload: ResponsesRequest) -> bool: return responses_payload_is_account_neutral_fresh_replay(payload.to_payload()) @@ -538,6 +559,7 @@ def stream_http_responses( downstream_turn_state: str | None = None, forwarded_request: bool = False, forwarded_original_request_unanchored: bool = False, + forwarded_proxy_injected_previous_response_id: bool = False, forwarded_legacy_signature: bool = False, forwarded_affinity_kind: str | None = None, forwarded_affinity_key: str | None = None, @@ -562,6 +584,7 @@ def stream_http_responses( downstream_turn_state=downstream_turn_state, forwarded_request=forwarded_request, forwarded_original_request_unanchored=forwarded_original_request_unanchored, + forwarded_proxy_injected_previous_response_id=forwarded_proxy_injected_previous_response_id, forwarded_legacy_signature=forwarded_legacy_signature, proxy_api_authorization=proxy_api_authorization, forwarded_affinity_kind=forwarded_affinity_kind, @@ -587,6 +610,7 @@ async def _stream_http_bridge_or_retry( downstream_turn_state: str | None = None, forwarded_request: bool = False, forwarded_original_request_unanchored: bool = False, + forwarded_proxy_injected_previous_response_id: bool = False, forwarded_legacy_signature: bool = False, proxy_api_authorization: str | None = None, forwarded_affinity_kind: str | None = None, @@ -676,6 +700,7 @@ async def _stream_http_bridge_or_retry( downstream_turn_state=downstream_turn_state, forwarded_request=forwarded_request, forwarded_original_request_unanchored=forwarded_original_request_unanchored, + forwarded_proxy_injected_previous_response_id=forwarded_proxy_injected_previous_response_id, forwarded_legacy_signature=forwarded_legacy_signature, proxy_api_authorization=proxy_api_authorization, forwarded_affinity_kind=forwarded_affinity_kind, @@ -713,6 +738,7 @@ async def _stream_via_http_bridge( downstream_turn_state: str | None = None, forwarded_request: bool = False, forwarded_original_request_unanchored: bool = False, + forwarded_proxy_injected_previous_response_id: bool = False, forwarded_legacy_signature: bool = False, proxy_api_authorization: str | None = None, forwarded_affinity_kind: str | None = None, @@ -825,6 +851,13 @@ def prepare_bridge_request( if not forwarded_request else None ) + durable_lookup_key_kind = bridge_session_key.affinity_kind + durable_lookup_key_value = bridge_session_key.affinity_key + durable_lookup_session_header = ( + session_header_fallback_key.affinity_key + if explicit_prompt_cache_key is not None and session_header_fallback_key is not None + else incoming_session_header + ) legacy_anchor_lookup = await _legacy_forward_anchor_lookup( durable_bridge=self._durable_bridge, bridge_session_key=bridge_session_key, @@ -848,15 +881,11 @@ def prepare_bridge_request( else: try: durable_lookup = await self._durable_bridge.lookup_request_targets( - session_key_kind=bridge_session_key.affinity_kind, - session_key_value=bridge_session_key.affinity_key, + session_key_kind=durable_lookup_key_kind, + session_key_value=durable_lookup_key_value, api_key_id=bridge_session_key.api_key_id, turn_state=durable_lookup_turn_state, - session_header=( - session_header_fallback_key.affinity_key - if explicit_prompt_cache_key is not None and session_header_fallback_key is not None - else incoming_session_header - ), + session_header=durable_lookup_session_header, previous_response_id=payload.previous_response_id, ) except ProxyResponseError: @@ -897,7 +926,7 @@ def prepare_bridge_request( durable_lookup = None effective_payload = payload untrimmed_effective_payload = payload - proxy_injected_previous_response_id = False + proxy_injected_previous_response_id = forwarded_proxy_injected_previous_response_id fresh_upstream_request_text: str | None = None previous_response_trimmed_input_count: int | None = None previous_response_trimmed_input_fingerprint: str | None = None @@ -907,13 +936,29 @@ def prepare_bridge_request( durable_full_resend_is_account_neutral: bool | None = None durable_full_resend_has_safe_fresh_context = False durable_full_resend_retains_prior_output = False + durable_compaction_recovery_allowed = False force_local_recovery_creation = False + live_local_session_exists = False + forwards_to_active_owner = False + fresh_reattach_requires_live_path = False payload_looks_like_full_resend = _http_bridge_payload_looks_like_full_resend(payload) + payload_is_same_account_compaction_recovery = responses_payload_is_same_account_compaction_recovery( + payload.to_payload() + ) def classify_durable_full_resend( lookup: DurableBridgeLookup, - ) -> tuple[int | None, str | None, bool]: + ) -> tuple[int | None, str | None, bool, bool]: stored_count = lookup.latest_input_item_count + compaction_recovery_allowed = ( + bridge_session_key.strength == "hard" + and lookup.account_id is not None + and isinstance(stored_count, int) + and stored_count > 0 + and isinstance(lookup.latest_input_full_fingerprint, str) + and bool(lookup.latest_input_full_fingerprint) + and payload_is_same_account_compaction_recovery + ) if ( not payload_looks_like_full_resend or stored_count is None @@ -924,7 +969,7 @@ def classify_durable_full_resend( ) or not isinstance(payload.input, list) ): - return None, None, False + return None, None, False, compaction_recovery_allowed replay_projection = project_responses_input_for_account_neutral_fresh_replay( cast(list[JsonValue], payload.input), stored_count=stored_count, @@ -942,15 +987,135 @@ def classify_durable_full_resend( pending_tool_calls=lookup.latest_pending_tool_calls, ) ) - return stored_count, lookup.latest_input_full_fingerprint, safe_fresh_context + return ( + stored_count, + lookup.latest_input_full_fingerprint, + safe_fresh_context, + compaction_recovery_allowed, + ) + + def durable_recovery_allows_unanchored_lineage() -> bool: + if durable_compaction_recovery_allowed or durable_full_resend_has_safe_fresh_context: + return True + if durable_full_resend_anchor_count is None or not isinstance(payload.input, list): + return False + replay_projection = project_responses_input_for_account_neutral_fresh_replay( + cast(list[JsonValue], payload.input), + stored_count=durable_full_resend_anchor_count, + ) + return replay_projection is not None and responses_input_has_self_contained_tool_continuation_suffix( + replay_projection.input_items, + stored_count=replay_projection.stored_prefix_count, + ) + + def bootstrap_refresh_allows_local_takeover( + *, + forwarded_lookup: DurableBridgeLookup | None, + refreshed_lookup: DurableBridgeLookup | None, + ) -> bool: + if forwarded_lookup == refreshed_lookup: + return True + if forwarded_lookup is None or refreshed_lookup is None: + return False + same_owner_snapshot = ( + forwarded_lookup.session_id == refreshed_lookup.session_id + and forwarded_lookup.canonical_kind == refreshed_lookup.canonical_kind + and forwarded_lookup.canonical_key == refreshed_lookup.canonical_key + and forwarded_lookup.api_key_scope == refreshed_lookup.api_key_scope + and forwarded_lookup.account_id is not None + and forwarded_lookup.account_id == refreshed_lookup.account_id + and forwarded_lookup.owner_instance_id == refreshed_lookup.owner_instance_id + and forwarded_lookup.owner_epoch == refreshed_lookup.owner_epoch + and forwarded_lookup.lease_expires_at == refreshed_lookup.lease_expires_at + and forwarded_lookup.state == refreshed_lookup.state + and forwarded_lookup.latest_turn_state == refreshed_lookup.latest_turn_state + and forwarded_lookup.latest_input_item_count == refreshed_lookup.latest_input_item_count + and forwarded_lookup.latest_input_full_fingerprint == refreshed_lookup.latest_input_full_fingerprint + and forwarded_lookup.model == refreshed_lookup.model + ) + return ( + same_owner_snapshot + and forwarded_lookup.latest_response_id is not None + and refreshed_lookup.latest_response_anchor_quarantined + and durable_recovery_allows_unanchored_lineage() + ) + + def enforce_quarantined_anchor_admission( + *, + lookup: DurableBridgeLookup, + request_payload: ResponsesRequest, + path: str, + ) -> None: + if ( + bridge_session_key.strength != "hard" + or not lookup.latest_response_anchor_quarantined + or request_payload.previous_response_id is not None + or bool(request_payload.conversation) + ): + return + if durable_recovery_allows_unanchored_lineage(): + _log_http_bridge_event( + "durable_anchor_quarantine_recovery_allowed", + bridge_session_key, + account_id=lookup.account_id, + model=request_payload.model, + detail=f"path={path}, outcome=verified_full_resend", + cache_key_family=bridge_session_key.affinity_kind, + model_class=_extract_model_class(request_payload.model) if request_payload.model else None, + owner_check_applied=True, + ) + return + _record_continuity_fail_closed( + surface="http_bridge", + reason="quarantined_anchor_requires_full_resend", + previous_response_id=None, + session_id=incoming_turn_state_header or incoming_session_header, + upstream_error_code="durable_anchor_quarantined", + ) + _log_http_bridge_event( + "durable_anchor_quarantine_recovery_rejected", + bridge_session_key, + account_id=lookup.account_id, + model=request_payload.model, + detail=f"path={path}, outcome=incremental_or_unverified_full_resend", + cache_key_family=bridge_session_key.affinity_kind, + model_class=_extract_model_class(request_payload.model) if request_payload.model else None, + owner_check_applied=True, + ) + raise _http_bridge_full_resend_required_error() + + def reject_fresh_socket_durable_anchor(*, lookup: DurableBridgeLookup, path: str) -> None: + _record_continuity_fail_closed( + surface="http_bridge", + reason="fresh_socket_anchor_requires_full_resend", + previous_response_id=None, + session_id=incoming_turn_state_header or incoming_session_header, + upstream_error_code="durable_anchor_from_prior_socket", + ) + _log_http_bridge_event( + "fresh_socket_durable_anchor_recovery_rejected", + bridge_session_key, + account_id=lookup.account_id, + model=payload.model, + detail=f"path={path}, outcome=incremental_or_unverified_full_resend", + cache_key_family=bridge_session_key.affinity_kind, + model_class=_extract_model_class(payload.model) if payload.model else None, + owner_check_applied=True, + ) + raise _http_bridge_full_resend_required_error() if durable_lookup is not None: ( durable_full_resend_anchor_count, durable_full_resend_anchor_fingerprint, durable_full_resend_has_safe_fresh_context, + durable_compaction_recovery_allowed, ) = classify_durable_full_resend(durable_lookup) - durable_anchor_trimmable = durable_full_resend_anchor_count is not None + enforce_quarantined_anchor_admission( + lookup=durable_lookup, + request_payload=payload, + path="initial_lookup", + ) durable_model_transition_lookup = ( durable_lookup if durable_lookup is not None and not _http_bridge_models_compatible(durable_lookup.model, payload.model) @@ -1006,22 +1171,32 @@ def classify_durable_full_resend( durable_lookup=durable_lookup, ) forwards_to_active_owner = await self._http_bridge_can_forward_to_active_owner(durable_lookup) - fresh_reattach_can_use_durable_anchor = ( + fresh_reattach_requires_live_path = ( + payload.previous_response_id is None + and not payload.conversation + and bridge_session_key.strength == "hard" + and durable_lookup.latest_response_id is not None + and not durable_recovery_allows_unanchored_lineage() + ) + fresh_reattach_has_previous_socket_anchor = ( not live_local_session_exists and not forwards_to_active_owner and payload.previous_response_id is None and not payload.conversation and bridge_session_key.strength == "hard" and durable_lookup.latest_response_id is not None - and (not payload_looks_like_full_resend or durable_anchor_trimmable) ) - if ( - fresh_reattach_can_use_durable_anchor - and payload_looks_like_full_resend - and durable_full_resend_has_safe_fresh_context - ): - # The client already supplied a complete fresh request. Adding - # a durable anchor here can strand it on the new WebSocket. + if fresh_reattach_has_previous_socket_anchor: + if not durable_recovery_allows_unanchored_lineage(): + # Preserve request preparation/validation evidence while + # keeping the retained connection-local id out of the + # serialized frame. No upstream transport exists yet. + _fresh_request_state, fresh_upstream_request_text = prepare_bridge_request(payload) + del _fresh_request_state + reject_fresh_socket_durable_anchor( + lookup=durable_lookup, + path="direct_reattach", + ) _log_http_bridge_event( "fresh_reattach_full_resend_preserved", bridge_session_key, @@ -1031,22 +1206,6 @@ def classify_durable_full_resend( cache_key_family=bridge_session_key.affinity_kind, model_class=_extract_model_class(payload.model) if payload.model else None, ) - elif fresh_reattach_can_use_durable_anchor: - effective_payload = payload.model_copy( - update={"previous_response_id": durable_lookup.latest_response_id} - ) - proxy_injected_previous_response_id = True - _fresh_request_state, fresh_upstream_request_text = prepare_bridge_request(payload) - del _fresh_request_state - _log_http_bridge_event( - "fresh_reattach_anchor_injected", - bridge_session_key, - account_id=None, - model=payload.model, - detail=f"response_id={durable_lookup.latest_response_id}", - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(payload.model) if payload.model else None, - ) account_neutral_recovery = is_http_bridge_account_neutral_replay( kind=bridge_session_key.affinity_kind, key=bridge_session_key.affinity_key, @@ -1152,10 +1311,14 @@ def classify_durable_full_resend( or model_transition_owner_missing ) continuity_preferred_account_id = request_state.preferred_account_id + request_state.account_bound_owner_id = ( + durable_lookup.account_id if durable_compaction_recovery_allowed and durable_lookup is not None else None + ) # Existing bridge/response ownership and file ownership are equally # hard. Merge them before transport creation; source ordering must not # turn a conflict into an implicit account switch. request_state.preferred_account_id = resolve_required_account_id( + ("account-bound compaction", request_state.account_bound_owner_id), ("previous response or bridge", request_state.preferred_account_id), ("input file", rewritten_file_account_id), ) @@ -1225,6 +1388,7 @@ def switch_to_account_neutral_replay() -> None: nonlocal account_neutral_recovery nonlocal affinity nonlocal bridge_session_key + nonlocal durable_compaction_recovery_allowed nonlocal durable_full_resend_anchor_count nonlocal durable_full_resend_anchor_fingerprint nonlocal durable_full_resend_fresh_payload @@ -1290,6 +1454,7 @@ def switch_to_account_neutral_replay() -> None: durable_full_resend_anchor_fingerprint = None durable_full_resend_fresh_payload = None durable_full_resend_is_account_neutral = None + durable_compaction_recovery_allowed = False durable_lookup = None file_required_preferred_account = False @@ -1335,6 +1500,7 @@ def switch_to_account_neutral_replay() -> None: forwarded_original_request_unanchored=original_request_unanchored, forwarded_affinity_kind=forwarded_affinity_kind, forwarded_affinity_key=forwarded_affinity_key, + allow_fresh_session_creation=not fresh_reattach_requires_live_path, durable_lookup=durable_lookup, request_stage=request_state.request_stage, preferred_account_id=request_state.preferred_account_id, @@ -1419,6 +1585,7 @@ def switch_to_account_neutral_replay() -> None: request_started_at=request_state.started_at, proxy_api_authorization=proxy_api_authorization, client_ip=client_ip, + proxy_injected_previous_response_id=proxy_injected_previous_response_id, ): forwarded_any = True yield line @@ -1450,6 +1617,61 @@ def switch_to_account_neutral_replay() -> None: previous_response_id=effective_payload.previous_response_id, ) ) + if should_attempt_bootstrap_rebind: + forwarded_bootstrap_lookup = durable_lookup + try: + refreshed_bootstrap_lookup = await self._durable_bridge.lookup_request_targets( + session_key_kind=durable_lookup_key_kind, + session_key_value=durable_lookup_key_value, + api_key_id=bridge_session_key.api_key_id, + turn_state=durable_lookup_turn_state, + session_header=durable_lookup_session_header, + previous_response_id=payload.previous_response_id, + ) + except ProxyResponseError: + raise + except Exception: + logger.warning( + "Bootstrap takeover lookup failed after owner forward failure; failing closed", + exc_info=True, + ) + should_attempt_bootstrap_rebind = False + else: + durable_lookup = refreshed_bootstrap_lookup + if refreshed_bootstrap_lookup is None: + durable_compaction_recovery_allowed = False + durable_full_resend_anchor_count = None + durable_full_resend_anchor_fingerprint = None + durable_full_resend_has_safe_fresh_context = False + else: + ( + durable_full_resend_anchor_count, + durable_full_resend_anchor_fingerprint, + durable_full_resend_has_safe_fresh_context, + durable_compaction_recovery_allowed, + ) = classify_durable_full_resend(refreshed_bootstrap_lookup) + enforce_quarantined_anchor_admission( + lookup=refreshed_bootstrap_lookup, + request_payload=effective_payload, + path="owner_forward_bootstrap_refresh", + ) + continuity_preferred_account_id = refreshed_bootstrap_lookup.account_id + request_state.preferred_account_id = resolve_required_account_id( + ("account-bound compaction", request_state.account_bound_owner_id), + ( + "refreshed bootstrap bridge", + continuity_preferred_account_id, + ), + ("input file", rewritten_file_account_id), + ) + preferred_account_has_continuity_provenance = ( + continuity_preferred_account_id is not None + and request_state.preferred_account_id == continuity_preferred_account_id + ) + should_attempt_bootstrap_rebind = bootstrap_refresh_allows_local_takeover( + forwarded_lookup=forwarded_bootstrap_lookup, + refreshed_lookup=refreshed_bootstrap_lookup, + ) should_attempt_turn_state_takeover = False if ( not owner_forward_fresh_replay @@ -1489,6 +1711,7 @@ def switch_to_account_neutral_replay() -> None: if _http_bridge_durable_lookup_allows_turn_state_takeover(fresh_turn_state_lookup): durable_lookup = fresh_turn_state_lookup if fresh_turn_state_lookup is None: + durable_compaction_recovery_allowed = False durable_full_resend_anchor_count = None durable_full_resend_anchor_fingerprint = None durable_full_resend_has_safe_fresh_context = False @@ -1497,9 +1720,16 @@ def switch_to_account_neutral_replay() -> None: durable_full_resend_anchor_count, durable_full_resend_anchor_fingerprint, durable_full_resend_has_safe_fresh_context, + durable_compaction_recovery_allowed, ) = classify_durable_full_resend(fresh_turn_state_lookup) + enforce_quarantined_anchor_admission( + lookup=fresh_turn_state_lookup, + request_payload=effective_payload, + path="owner_forward_refresh", + ) continuity_preferred_account_id = fresh_turn_state_lookup.account_id request_state.preferred_account_id = resolve_required_account_id( + ("account-bound compaction", request_state.account_bound_owner_id), ( "refreshed previous response or bridge", continuity_preferred_account_id, @@ -1520,10 +1750,15 @@ def switch_to_account_neutral_replay() -> None: or ( fresh_turn_state_lookup is not None and fresh_turn_state_lookup.account_id is not None - and durable_full_resend_anchor_count is not None and ( - durable_full_resend_has_safe_fresh_context - or fresh_turn_state_lookup.latest_response_id is not None + durable_compaction_recovery_allowed + or ( + durable_full_resend_anchor_count is not None + and ( + durable_recovery_allows_unanchored_lineage() + or fresh_turn_state_lookup.latest_response_id is not None + ) + ) ) ) ) @@ -1566,6 +1801,19 @@ def switch_to_account_neutral_replay() -> None: model_class=_extract_model_class(effective_payload.model) if effective_payload.model else None, owner_check_applied=True, ) + if ( + not owner_forward_fresh_replay + and effective_payload.previous_response_id is None + and not effective_payload.conversation + and bridge_session_key.strength == "hard" + and durable_lookup is not None + and durable_lookup.latest_response_id is not None + and not durable_recovery_allows_unanchored_lineage() + ): + reject_fresh_socket_durable_anchor( + lookup=durable_lookup, + path="owner_forward_recovery", + ) while True: try: session = await self._get_or_create_http_bridge_session( @@ -1605,6 +1853,7 @@ def switch_to_account_neutral_replay() -> None: ), preferred_account_id=request_state.preferred_account_id, preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, + fallback_on_preferred_account_unavailable=(request_state.account_bound_owner_id is None), request_usage_budget=request_state.request_usage_budget, session_header_fallback_key=session_header_fallback_key, request_deadline=request_deadline, @@ -1655,53 +1904,11 @@ def switch_to_account_neutral_replay() -> None: ), outcome="success", ) - # Best-effort synthetic interrupted-output injection for the - # local recovery request. The pending tool-call metadata lives - # in the owning instance's in-memory session state, so after an - # owner-forward failure it is only available when the rebound - # local session still carries it (for example when ownership - # flapped back to this instance). A fresh local rebind cannot - # know the interrupted call ids; in that case the anchored - # request is resubmitted unmodified (matching pre-injection - # behavior) and an upstream missing-tool-output error is - # classified and masked as a retryable continuity failure. + # Best-effort synthetic interrupted-output injection remains + # available only when the rebound session itself carries + # current-socket pending-call state. A durable id from the + # failed owner's socket is never copied into this fresh one. recovery_payload = effective_payload - recovery_anchor_input_count: int | None = None - recovery_anchor_input_fingerprint: str | None = None - if ( - not owner_forward_fresh_replay - and not durable_full_resend_has_safe_fresh_context - and recovery_payload.previous_response_id is None - and durable_lookup is not None - and durable_lookup.latest_response_id is not None - and durable_full_resend_anchor_count is not None - and durable_full_resend_anchor_fingerprint is not None - and isinstance(recovery_payload.input, list) - and len(recovery_payload.input) > durable_full_resend_anchor_count - ): - recovery_input = cast(list[JsonValue], recovery_payload.input) - recovery_anchor_input_count = len(recovery_input) - recovery_anchor_input_fingerprint = _fingerprint_input_items(recovery_input) - recovery_payload = recovery_payload.model_copy( - update={ - "previous_response_id": durable_lookup.latest_response_id, - "input": recovery_input[durable_full_resend_anchor_count:], - } - ) - if durable_lookup.latest_response_id != session.last_completed_response_id: - session.last_pending_tool_calls = {} - session.last_completed_response_id = durable_lookup.latest_response_id - session.last_completed_input_count = durable_full_resend_anchor_count - session.last_completed_input_prefix_fingerprint = durable_full_resend_anchor_fingerprint - _log_http_bridge_event( - "owner_forward_recovery_anchor_injected", - bridge_session_key, - account_id=durable_lookup.account_id, - model=recovery_payload.model, - detail=f"response_id={durable_lookup.latest_response_id}", - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(recovery_payload.model) if recovery_payload.model else None, - ) recovery_injected_input = _http_bridge_interrupted_tool_outputs_input( session, payload=recovery_payload, @@ -1744,12 +1951,10 @@ def switch_to_account_neutral_replay() -> None: ) retry_request_state.preferred_account_id = request_state.preferred_account_id retry_request_state.excluded_account_ids.update(request_state.excluded_account_ids) - if recovery_anchor_input_count is not None: - retry_request_state.input_item_count = recovery_anchor_input_count - retry_request_state.input_full_fingerprint = recovery_anchor_input_fingerprint - retry_request_state.proxy_injected_previous_response_id = True - retry_request_state.fresh_upstream_request_is_retry_safe = False - + _preserve_proxy_anchor_recovery_state( + request_state, + retry_request_state, + ) async for event_block in self._stream_http_bridge_session_events( session, request_state=retry_request_state, @@ -1776,8 +1981,28 @@ def switch_to_account_neutral_replay() -> None: session.last_used_at = _service_time().monotonic() return session = session_or_forward + resolved_current_socket_anchor = ( + durable_lookup is not None + and durable_lookup.latest_response_id is not None + and session.last_completed_response_id == durable_lookup.latest_response_id + and session.last_completed_response_store is False + ) if ( - not durable_full_resend_has_safe_fresh_context + effective_payload.previous_response_id is None + and not effective_payload.conversation + and bridge_session_key.strength == "hard" + and durable_lookup is not None + and durable_lookup.latest_response_id is not None + and not resolved_current_socket_anchor + and not durable_recovery_allows_unanchored_lineage() + ): + reject_fresh_socket_durable_anchor( + lookup=durable_lookup, + path="session_resolution_race", + ) + if ( + live_local_session_exists + and not durable_full_resend_has_safe_fresh_context and durable_full_resend_anchor_count is not None and durable_full_resend_anchor_fingerprint is not None and durable_lookup is not None @@ -1786,8 +2011,10 @@ def switch_to_account_neutral_replay() -> None: if durable_lookup.latest_response_id != session.last_completed_response_id: # The pending tool calls were recorded for the session's own # last completed response; a durable anchor pointing elsewhere - # must not trigger interrupted-output injection. + # must not trigger interrupted-output injection or inherit + # current-socket store provenance from that response. session.last_pending_tool_calls = {} + session.last_completed_response_store = None session.last_completed_response_id = durable_lookup.latest_response_id session.last_completed_input_count = durable_full_resend_anchor_count session.last_completed_input_prefix_fingerprint = durable_full_resend_anchor_fingerprint @@ -1826,6 +2053,7 @@ def switch_to_account_neutral_replay() -> None: and not proxy_injected_previous_response_id and effective_payload.previous_response_id is None and session.last_completed_response_id is not None + and session.last_completed_response_store is False and (session_anchor_trimmable or recovery_session_can_anchor) ): fresh_upstream_request_text = text_data @@ -1925,6 +2153,7 @@ def switch_to_account_neutral_replay() -> None: durable_lookup=durable_lookup, ) request_state.preferred_account_id = previous_request_state.preferred_account_id + request_state.account_bound_owner_id = previous_request_state.account_bound_owner_id request_state.excluded_account_ids.update(previous_request_state.excluded_account_ids) if store_context_trim_applied: # Store the full incoming client input as the session context @@ -1946,7 +2175,7 @@ def switch_to_account_neutral_replay() -> None: # keep the replay-safety decision made when the anchor was # injected. request_state.fresh_upstream_request_is_retry_safe = ( - (durable_full_resend_anchor_count is None or durable_full_resend_has_safe_fresh_context) + (durable_full_resend_anchor_count is None or durable_recovery_allows_unanchored_lineage()) if store_context_trim_applied else previous_request_state.fresh_upstream_request_is_retry_safe ) @@ -2035,7 +2264,9 @@ def switch_to_account_neutral_replay() -> None: preferred_account_id=replacement_preferred_account_id, preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, fallback_on_preferred_account_unavailable=not ( - file_required_preferred_account or request_state.previous_response_id is not None + file_required_preferred_account + or request_state.previous_response_id is not None + or request_state.account_bound_owner_id is not None ), allow_previous_response_recovery_rebind=request_state.previous_response_id is not None, request_usage_budget=request_state.request_usage_budget, @@ -2330,7 +2561,8 @@ def switch_to_account_neutral_replay() -> None: preferred_account_id=retry_preferred_account_id, preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, fallback_on_preferred_account_unavailable=not ( - file_required_preferred_account and retry_preferred_account_id is not None + (file_required_preferred_account and retry_preferred_account_id is not None) + or request_state.account_bound_owner_id is not None ), request_usage_budget=estimate_api_key_request_usage(retry_payload), request_deadline=request_deadline, @@ -2396,6 +2628,10 @@ def switch_to_account_neutral_replay() -> None: retry_request_state.request_stage = retry_request_stage retry_request_state.preferred_account_id = retry_preferred_account_id retry_request_state.excluded_account_ids.update(request_state.excluded_account_ids) + _preserve_proxy_anchor_recovery_state( + request_state, + retry_request_state, + ) retry_events: AsyncGenerator[str, None] = self._stream_http_bridge_session_events( session, diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index a70367af9b..6f6762e494 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -27,7 +27,11 @@ from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401 from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401 from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401 -from app.core.clients.proxy_websocket import UpstreamWebSocketMessage, UpstreamWebSocketTransportError +from app.core.clients.proxy_websocket import ( + UpstreamWebSocketMessage, + UpstreamWebSocketTransportError, + await_pending_websocket_receive_classification, +) from app.core.errors import response_failed_event from app.core.openai.parsing import parse_sse_event_payload from app.core.types import JsonValue @@ -51,7 +55,10 @@ _http_bridge_request_counts_against_queue, _log_http_bridge_event, _normalize_http_bridge_error_event, + _quarantine_http_bridge_disconnected_socket_anchor, + _quarantine_http_bridge_durable_anchor, _record_http_bridge_stuck_retire, + _select_http_bridge_disconnect_anchor_quarantine_response_id, ) from app.modules.proxy._service.http_bridge.service_stubs import ( _assign_websocket_response_id, @@ -536,8 +543,10 @@ async def _relay_http_bridge_upstream_messages( relay_upstream = session.upstream receive_task: asyncio.Task[UpstreamWebSocketMessage] | None = None wakeup_task: asyncio.Task[bool] | None = None + disconnect_quarantine_response_id: str | None = None try: while True: + disconnect_quarantine_response_id = None # Clear before taking the deadline snapshot. A send before the # clear is represented by its timestamp; a send after it leaves # the event set and wakes the persistent receive wait below. @@ -595,6 +604,8 @@ async def _relay_http_bridge_upstream_messages( wakeup_task = None if timed_out: + if await await_pending_websocket_receive_classification(receive_task): + continue if receive_timeout is None: raise RuntimeError("HTTP bridge reader timed out without a timeout contract") if receive_timeout.error_code == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL: @@ -610,18 +621,16 @@ async def _relay_http_bridge_upstream_messages( async with session.pending_lock: if receive_task is not None and receive_task.done(): continue - expired_owner = any( - deadline is not None and deadline <= now - for request_state in session.pending_requests - if ( - deadline := _http_bridge_eventless_precreated_deadline( - request_state, - stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds, - ) + expired_owner: _WebSocketRequestState | None = None + for request_state in session.pending_requests: + deadline = _http_bridge_eventless_precreated_deadline( + request_state, + stuck_gate_retire_after_seconds=stuck_gate_retire_after_seconds, ) - is not None - ) - if not expired_owner: + if deadline is not None and deadline <= now: + expired_owner = request_state + break + if expired_owner is None: continue pending_count = len(session.pending_requests) for request_state in session.pending_requests: @@ -641,6 +650,11 @@ async def _relay_http_bridge_upstream_messages( ) if receive_cancelled: receive_task = None + await _quarantine_http_bridge_durable_anchor( + self, + session, + request_state=expired_owner, + ) _record_http_bridge_stuck_retire( reason=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, session=session, @@ -702,8 +716,20 @@ async def _relay_http_bridge_upstream_messages( async with session.pending_lock: archive_request_state = session.pending_requests[0] if len(session.pending_requests) == 1 else None + disconnect_quarantine_response_id = _select_http_bridge_disconnect_anchor_quarantine_response_id( + session.pending_requests, + latest_response_id=session.last_completed_response_id, + latest_response_store=session.last_completed_response_store, + ) _archive_http_bridge_upstream_message(session, message, archive_request_state) session.last_upstream_close_code = message.close_code + if disconnect_quarantine_response_id is not None: + await _quarantine_http_bridge_disconnected_socket_anchor( + self, + session, + expected_response_id=disconnect_quarantine_response_id, + ) + disconnect_quarantine_response_id = None retried = False # A process-network receive failure follows a successful send; # replay is not safe merely because output is not visible. @@ -1450,22 +1476,24 @@ async def _process_http_bridge_upstream_text( completed_empty_prewarm = False if event_type == "response.completed" and terminal_request_state is not None and not completed_empty_prewarm: - # Record the completed response id regardless of input shape so - # subsequent turns (including ones that never populated - # input_item_count, e.g. string inputs) can still reuse this - # anchor for continuity lookups. - if response_id is not None: - session.last_completed_response_id = response_id - # Remember which tool-call items the completed response left - # pending so an anchored follow-up that omits their outputs - # (interrupted turn) can receive synthetic interrupted - # outputs instead of an upstream missing-tool-output 400. - session.last_pending_tool_calls = dict(terminal_request_state.pending_tool_call_types) - # Prefix trimming is only meaningful for list-shaped inputs, so - # keep the input-count / fingerprint update scoped to that path. - if terminal_request_state.input_item_count > 0: - session.last_completed_input_count = terminal_request_state.input_item_count - session.last_completed_input_prefix_fingerprint = terminal_request_state.input_full_fingerprint + async with session.lifecycle_lock: + # Record the completed response id regardless of input shape so + # subsequent turns (including ones that never populated + # input_item_count, e.g. string inputs) can still reuse this + # anchor for continuity lookups. + if response_id is not None: + session.last_completed_response_id = response_id + session.last_completed_response_store = terminal_request_state.response_store + # Remember which tool-call items the completed response left + # pending so an anchored follow-up that omits their outputs + # (interrupted turn) can receive synthetic interrupted + # outputs instead of an upstream missing-tool-output 400. + session.last_pending_tool_calls = dict(terminal_request_state.pending_tool_call_types) + # Prefix trimming is only meaningful for list-shaped inputs, so + # keep the input-count / fingerprint update scoped to that path. + if terminal_request_state.input_item_count > 0: + session.last_completed_input_count = terminal_request_state.input_item_count + session.last_completed_input_prefix_fingerprint = terminal_request_state.input_full_fingerprint normalize_error_event = ( terminal_request_state is None or terminal_request_state.enforce_openai_sdk_contract diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 0d842e78f9..192cb380b4 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -798,6 +798,10 @@ class _WebSocketRequestState: precreated_replay_account_id: str | None = None skip_request_log: bool = False previous_response_id: str | None = None + # Persisted-response intent from the validated response.create payload. + # ``None`` means the state was constructed outside the normal preparation + # path and is not sufficient proof for disconnect-time invalidation. + response_store: bool | None = None session_id: str | None = None proxy_injected_previous_response_id: bool = False expose_stale_previous_response_classifier: bool = False @@ -819,6 +823,11 @@ class _WebSocketRequestState: fresh_upstream_request_responses_lite_model: str | None = None request_stage: str = "first_turn" preferred_account_id: str | None = None + # Exact upstream owner for request bodies that are safe only on one + # account, such as encrypted Codex compaction context. Unlike ordinary + # preference this survives reconnect/retry preparation and forbids + # account-switch fallbacks. + account_bound_owner_id: str | None = None require_security_work_authorized: bool = False file_required_preferred_account: bool = False bridge_soft_capacity_reroute_allowed: bool = False @@ -939,6 +948,10 @@ class _HTTPBridgeSession: previous_response_alias_registration_generations: dict[str, int] = field(default_factory=dict) last_completed_input_count: int = 0 last_completed_response_id: str | None = None + # Effective store intent for ``last_completed_response_id`` only when that + # response completed on the session's current upstream socket. Reconnects + # clear this without discarding durable routing or prefix proof. + last_completed_response_store: bool | None = None last_completed_input_prefix_fingerprint: str | None = None last_pending_tool_calls: dict[str, str] = field(default_factory=dict) durable_session_id: str | None = None diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 7201fa13ec..7a1e841f38 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -381,6 +381,8 @@ def _prepare_websocket_request_state_for_account_switch( request_state: "_WebSocketRequestState", ) -> str | None: """Return an unsent request body only when moving accounts is proven safe.""" + if request_state.account_bound_owner_id is not None: + return None if request_state.previous_response_id is None: return request_state.request_text if not ( @@ -805,7 +807,7 @@ def _websocket_fresh_request_blocks_account_switch(request_state: _WebSocketRequ def _websocket_auth_request_can_switch_account(request_state: _WebSocketRequestState) -> bool: - if request_state.file_required_preferred_account: + if request_state.file_required_preferred_account or request_state.account_bound_owner_id is not None: return False if request_state.previous_response_id is None: return True @@ -823,7 +825,10 @@ def _prepare_websocket_request_state_for_auth_replay( ) -> str | None: if request_state.last_downstream_sequence_number is not None: return None - if not _websocket_auth_request_can_switch_account(request_state): + account_bound_replay = request_state.account_bound_owner_id is not None + if not account_bound_replay and not _websocket_auth_request_can_switch_account(request_state): + return None + if account_bound_replay and request_state.previous_response_id is not None: return None if ( request_state.proxy_injected_previous_response_id diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index fa7fd7d8ee..fe69fb2270 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -57,7 +57,9 @@ from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401 from app.core.clients.proxy_websocket import ( UpstreamWebSocket, + UpstreamWebSocketMessage, UpstreamWebSocketTransportError, + await_pending_websocket_receive_classification, filter_inbound_websocket_headers, ) from app.core.errors import ( @@ -3235,6 +3237,7 @@ async def _relay_upstream_websocket_messages( ) -> None: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy + receive_task: asyncio.Task[UpstreamWebSocketMessage] | None = None try: while True: receive_timeout = await proxy._next_websocket_receive_timeout( @@ -3248,18 +3251,31 @@ async def _relay_upstream_websocket_messages( ) try: while True: - wait_timeout = None if receive_deadline is None else receive_deadline - time.monotonic() - if wait_timeout is not None and wait_timeout <= 0: - raise asyncio.TimeoutError() - keepalive_interval = getattr(_facade().get_settings(), "sse_keepalive_interval_seconds", 10.0) - if keepalive_interval > 0: - wait_timeout = ( - keepalive_interval if wait_timeout is None else min(wait_timeout, keepalive_interval) + if receive_task is not None and receive_task.done(): + message = receive_task.result() + receive_task = None + else: + wait_timeout = None if receive_deadline is None else receive_deadline - time.monotonic() + if wait_timeout is not None and wait_timeout <= 0: + raise asyncio.TimeoutError() + keepalive_interval = getattr( + _facade().get_settings(), + "sse_keepalive_interval_seconds", + 10.0, ) - message = await asyncio.wait_for( - upstream.receive(), - timeout=wait_timeout, - ) + if keepalive_interval > 0: + wait_timeout = ( + keepalive_interval + if wait_timeout is None + else min(wait_timeout, keepalive_interval) + ) + if receive_task is None: + receive_task = asyncio.create_task(upstream.receive()) + message = await asyncio.wait_for( + asyncio.shield(receive_task), + timeout=wait_timeout, + ) + receive_task = None archive_request_id = await _websocket_archive_request_id_for_message( message, pending_requests=pending_requests, @@ -3306,6 +3322,8 @@ async def _relay_upstream_websocket_messages( ) break continue + if await await_pending_websocket_receive_classification(receive_task): + continue if receive_timeout is None: raise if receive_timeout.fail_all_pending: @@ -3580,6 +3598,17 @@ async def _relay_upstream_websocket_messages( downstream_activity=downstream_activity, ) finally: + if receive_task is not None: + try: + await _facade()._await_cancelled_task( + receive_task, + label="proxy websocket upstream receive", + ) + except Exception: + _facade().logger.debug( + "Failed to cancel proxy websocket upstream receive", + exc_info=True, + ) async with pending_lock: has_pending_requests = bool(pending_requests) if not upstream_control.reconnect_requested and has_pending_requests: diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index fcbc012a8a..1e527fbcc5 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -1260,6 +1260,9 @@ async def internal_bridge_responses( include_rate_limit_headers=False, forwarded_request=True, forwarded_original_request_unanchored=forwarded_request_context.context.original_request_unanchored, + forwarded_proxy_injected_previous_response_id=( + forwarded_request_context.context.proxy_injected_previous_response_id + ), forwarded_legacy_signature=forwarded_request_context.context.signature_version is None, forwarded_headers=forwarded_headers, forwarded_downstream_turn_state=forwarded_request_context.context.downstream_turn_state, @@ -4796,6 +4799,7 @@ async def _stream_responses( include_rate_limit_headers: bool = True, forwarded_request: bool = False, forwarded_original_request_unanchored: bool = False, + forwarded_proxy_injected_previous_response_id: bool = False, forwarded_legacy_signature: bool = False, forwarded_headers: Mapping[str, str] | None = None, forwarded_downstream_turn_state: str | None = None, @@ -4966,6 +4970,7 @@ async def _stream_responses( downstream_turn_state=downstream_turn_state, forwarded_request=forwarded_request, forwarded_original_request_unanchored=forwarded_original_request_unanchored, + forwarded_proxy_injected_previous_response_id=forwarded_proxy_injected_previous_response_id, forwarded_legacy_signature=forwarded_legacy_signature, forwarded_affinity_kind=forwarded_affinity_kind, forwarded_affinity_key=forwarded_affinity_key, diff --git a/app/modules/proxy/durable_bridge_coordinator.py b/app/modules/proxy/durable_bridge_coordinator.py index 75e397a7d2..8cdf0c654e 100644 --- a/app/modules/proxy/durable_bridge_coordinator.py +++ b/app/modules/proxy/durable_bridge_coordinator.py @@ -43,6 +43,14 @@ class DurableBridgeLookup: model: str | None = None latest_pending_tool_calls: dict[str, str] | None = None + @property + def latest_response_anchor_quarantined(self) -> bool: + return ( + self.latest_response_id is None + and self.latest_input_item_count is not None + and self.latest_input_full_fingerprint is not None + ) + def lease_is_active(self, *, now: datetime) -> bool: if self.owner_instance_id is None: return False @@ -280,6 +288,25 @@ async def release_live_session( return None return _to_lookup(snapshot) + async def clear_latest_response_anchor_if_current( + self, + *, + session_id: str, + instance_id: str, + owner_epoch: int, + expected_response_id: str, + ) -> DurableBridgeLookup | None: + async with self._session() as session: + snapshot = await DurableBridgeRepository(session).clear_latest_response_anchor_if_current( + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + expected_response_id=expected_response_id, + ) + if snapshot is None: + return None + return _to_lookup(snapshot) + async def mark_instance_draining(self, *, instance_id: str) -> int: async with self._session() as session: return await DurableBridgeRepository(session).mark_owner_draining(instance_id=instance_id) diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index 7b82e7f8a3..453bc54aff 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -30,6 +30,10 @@ ) _PURGE_CLOSED_BATCH_SIZE = 500 _SESSION_ID_LOOKUP_CHUNK_SIZE = 500 +_DURABLE_ANCHOR_QUARANTINE_SENTINEL_COUNT = -1 +_DURABLE_ANCHOR_QUARANTINE_SENTINEL_FINGERPRINT = sha256( + b"codex-lb:durable-anchor-quarantined-without-prefix-proof" +).hexdigest() class DurableBridgeAliasRegistration(StrEnum): @@ -406,6 +410,52 @@ async def release_session( values=values, ) + async def clear_latest_response_anchor_if_current( + self, + *, + session_id: str, + instance_id: str, + owner_epoch: int, + expected_response_id: str, + ) -> DurableBridgeSessionSnapshot | None: + """Clear one exact durable recovery anchor with owner fencing. + + The previous-response alias remains registered for explicit continuity + lookup. The automatic latest anchor and pending-tool metadata are + quarantined while the input count and fingerprint remain as proof that + an unanchored recovery must be a verified full-context resend. + """ + + has_usable_input_proof = and_( + HttpBridgeSessionRecord.latest_input_item_count > 0, + HttpBridgeSessionRecord.latest_input_full_fingerprint.is_not(None), + HttpBridgeSessionRecord.latest_input_full_fingerprint != "", + ) + return await self._execute_fenced_session_update( + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + expected_latest_response_id=expected_response_id, + values={ + "latest_response_id": None, + "latest_input_item_count": case( + ( + has_usable_input_proof, + HttpBridgeSessionRecord.latest_input_item_count, + ), + else_=_DURABLE_ANCHOR_QUARANTINE_SENTINEL_COUNT, + ), + "latest_input_full_fingerprint": case( + ( + has_usable_input_proof, + HttpBridgeSessionRecord.latest_input_full_fingerprint, + ), + else_=_DURABLE_ANCHOR_QUARANTINE_SENTINEL_FINGERPRINT, + ), + "latest_pending_tool_calls_json": None, + }, + ) + async def _execute_fenced_session_update( self, *, @@ -413,17 +463,18 @@ async def _execute_fenced_session_update( instance_id: str, owner_epoch: int, values: dict[str, object], + expected_latest_response_id: str | None = None, ) -> DurableBridgeSessionSnapshot | None: + predicates = [ + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ] + if expected_latest_response_id is not None: + predicates.append(HttpBridgeSessionRecord.latest_response_id == expected_latest_response_id) async with sqlite_writer_section(): result = await self._session.execute( - update(HttpBridgeSessionRecord) - .where( - HttpBridgeSessionRecord.id == session_id, - HttpBridgeSessionRecord.owner_instance_id == instance_id, - HttpBridgeSessionRecord.owner_epoch == owner_epoch, - ) - .values(**values) - .returning(*_SNAPSHOT_COLUMNS) + update(HttpBridgeSessionRecord).where(*predicates).values(**values).returning(*_SNAPSHOT_COLUMNS) ) updated_row = result.one_or_none() await self._session.commit() diff --git a/app/modules/proxy/http_bridge_forwarding.py b/app/modules/proxy/http_bridge_forwarding.py index 7f8969da47..f1d4a655d2 100644 --- a/app/modules/proxy/http_bridge_forwarding.py +++ b/app/modules/proxy/http_bridge_forwarding.py @@ -57,6 +57,7 @@ HTTP_BRIDGE_AFFINITY_KEY_HEADER = "x-codex-bridge-affinity-key" HTTP_BRIDGE_FILE_OWNER_HEADER = "x-codex-bridge-file-owner" HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER = "x-codex-bridge-original-unanchored" +HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER = "x-codex-bridge-proxy-injected-previous-response" HTTP_BRIDGE_SIGNATURE_VERSION_HEADER = "x-codex-bridge-signature-version" HTTP_BRIDGE_CLIENT_IP_HEADER = "x-codex-bridge-client-ip" HTTP_BRIDGE_CLIENT_IP_SIGNATURE_HEADER = "x-codex-bridge-client-ip-signature" @@ -82,6 +83,7 @@ class HTTPBridgeForwardContext: codex_session_affinity: bool downstream_turn_state: str | None original_request_unanchored: bool = False + proxy_injected_previous_response_id: bool = False original_affinity_kind: str | None = None original_affinity_key: str | None = None file_owner_account_id: str | None = None @@ -222,10 +224,16 @@ def build_owner_forward_headers( forwarded[HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER] = context.origin_instance forwarded[HTTP_BRIDGE_TARGET_INSTANCE_HEADER] = context.target_instance forwarded[HTTP_BRIDGE_CODEX_AFFINITY_HEADER] = "1" if context.codex_session_affinity else "0" - signature_version = _HTTP_BRIDGE_SIGNATURE_VERSION_V2 if context.original_request_unanchored else None + signature_version = ( + _HTTP_BRIDGE_SIGNATURE_VERSION_V2 + if context.original_request_unanchored or context.proxy_injected_previous_response_id + else None + ) if signature_version is not None: forwarded[HTTP_BRIDGE_SIGNATURE_VERSION_HEADER] = signature_version - forwarded[HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER] = "1" + forwarded[HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER] = "1" if context.original_request_unanchored else "0" + if context.proxy_injected_previous_response_id: + forwarded[HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER] = "1" if context.original_affinity_kind and context.original_affinity_key: forwarded[HTTP_BRIDGE_AFFINITY_KIND_HEADER] = context.original_affinity_kind forwarded[HTTP_BRIDGE_AFFINITY_KEY_HEADER] = context.original_affinity_key @@ -300,6 +308,10 @@ def parse_forwarded_request( client_ip = _optional_header(headers.get(HTTP_BRIDGE_CLIENT_IP_HEADER)) signature_version = _optional_header(headers.get(HTTP_BRIDGE_SIGNATURE_VERSION_HEADER)) original_unanchored_value = _optional_header(headers.get(HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER)) + proxy_injected_value = _optional_header(headers.get(HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER)) + if proxy_injected_value not in {None, "1"}: + return None, _invalid_bridge_forward_signature_error() + proxy_injected_previous_response_id = proxy_injected_value == "1" if signature_version == _HTTP_BRIDGE_SIGNATURE_VERSION_V2: if original_unanchored_value not in {"0", "1"}: return None, _invalid_bridge_forward_signature_error() @@ -308,12 +320,17 @@ def parse_forwarded_request( original_request_unanchored = False else: return None, _invalid_bridge_forward_signature_error() + if proxy_injected_previous_response_id and ( + signature_version != _HTTP_BRIDGE_SIGNATURE_VERSION_V2 or payload.previous_response_id is None + ): + return None, _invalid_bridge_forward_signature_error() context = HTTPBridgeForwardContext( origin_instance=headers.get(HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER, "").strip() or "unknown", target_instance=target_instance, codex_session_affinity=_bool_header(headers.get(HTTP_BRIDGE_CODEX_AFFINITY_HEADER)), downstream_turn_state=_optional_header(headers.get("x-codex-turn-state")), original_request_unanchored=original_request_unanchored, + proxy_injected_previous_response_id=proxy_injected_previous_response_id, original_affinity_kind=_optional_header(headers.get(HTTP_BRIDGE_AFFINITY_KIND_HEADER)), original_affinity_key=_optional_header(headers.get(HTTP_BRIDGE_AFFINITY_KEY_HEADER)), file_owner_account_id=_optional_header(headers.get(HTTP_BRIDGE_FILE_OWNER_HEADER)), @@ -342,6 +359,11 @@ def parse_forwarded_request( ) if tools_bound_valid: return HTTPBridgeForwardedRequest(context=context), None + if context.proxy_injected_previous_response_id: + # Proxy-injected provenance controls durable quarantine. Never accept a + # forward that claims it through the rolling-upgrade primary fallback, + # whose legacy form does not authenticate this additive field. + return None, _invalid_bridge_forward_signature_error() if context.file_owner_account_id is not None or extract_input_file_ids(payload.input): # The rolling-upgrade primary signature does not bind the additive # file-owner proof. Never allow a stripped/forged proof to downgrade to @@ -559,32 +581,37 @@ def _structured_bridge_signing_payload( # Canonical structured encoding: object boundaries make field re-packing # impossible, the client-IP mode is itself authenticated, and ``protocol`` # domain-separates the primary and tamper-proofing signatures. + signing_context: JsonObject = { + "body_digest": body_digest, + "client_ip": context.client_ip if include_client_ip else None, + "client_ip_present": context.client_ip is not None, + "codex_session_affinity": context.codex_session_affinity, + "downstream_turn_state": context.downstream_turn_state, + "file_owner_account_id": context.file_owner_account_id, + "include_client_ip": include_client_ip, + "origin_instance": context.origin_instance, + "original_affinity_key": context.original_affinity_key, + "original_affinity_kind": context.original_affinity_kind, + "original_request_unanchored": context.original_request_unanchored, + "protocol": protocol, + "reservation": ( + { + "id": context.reservation.reservation_id, + "key_id": context.reservation.key_id, + "model": context.reservation.model, + } + if context.reservation is not None + else None + ), + "signature_version": signature_version, + "target_instance": context.target_instance, + } + if context.proxy_injected_previous_response_id: + # Additive encoding keeps false/absent forwards byte-compatible with + # pre-provenance replicas while binding the security-relevant true case. + signing_context["proxy_injected_previous_response_id"] = True return json.dumps( - { - "body_digest": body_digest, - "client_ip": context.client_ip if include_client_ip else None, - "client_ip_present": context.client_ip is not None, - "codex_session_affinity": context.codex_session_affinity, - "downstream_turn_state": context.downstream_turn_state, - "file_owner_account_id": context.file_owner_account_id, - "include_client_ip": include_client_ip, - "origin_instance": context.origin_instance, - "original_affinity_key": context.original_affinity_key, - "original_affinity_kind": context.original_affinity_kind, - "original_request_unanchored": context.original_request_unanchored, - "protocol": protocol, - "reservation": ( - { - "id": context.reservation.reservation_id, - "key_id": context.reservation.key_id, - "model": context.reservation.model, - } - if context.reservation is not None - else None - ), - "signature_version": signature_version, - "target_instance": context.target_instance, - }, + signing_context, ensure_ascii=True, sort_keys=True, separators=(",", ":"), diff --git a/app/modules/proxy/replay_safety.py b/app/modules/proxy/replay_safety.py index e350b1aa03..d6bd08c870 100644 --- a/app/modules/proxy/replay_safety.py +++ b/app/modules/proxy/replay_safety.py @@ -111,6 +111,7 @@ "x-openai-subagent", } ) +_SAME_ACCOUNT_COMPACTION_ITEM_FIELDS = frozenset({"encrypted_content", "id", "type"}) _ACCOUNT_SCOPED_HOSTED_INPUT_TYPES = frozenset( { "code_interpreter_call", @@ -241,6 +242,23 @@ def responses_input_items_are_self_contained_fresh_replay(input_items: list[Json return all(not call_ids for call_ids in unsettled_call_ids_by_type.values()) +def responses_input_has_self_contained_tool_continuation_suffix( + input_items: list[JsonValue], + *, + stored_count: int, +) -> bool: + """Prove that a full resend advances through a complete direct tool round trip.""" + + if stored_count <= 0 or len(input_items) <= stored_count: + return False + suffix = input_items[stored_count:] + return ( + responses_input_items_are_self_contained_fresh_replay(input_items) + and responses_payload_is_account_neutral_fresh_replay({"input": suffix}) + and any(isinstance(item, dict) and item.get("type") in _TOOL_CALL_TYPES for item in suffix) + ) + + def _internal_chat_message_metadata_is_account_neutral(value: JsonValue | None) -> bool: if value is None: return True @@ -540,6 +558,32 @@ def responses_payload_is_account_neutral_fresh_replay(payload: Mapping[str, Json return _tools_are_account_neutral(tools) +def responses_payload_is_same_account_compaction_recovery(payload: Mapping[str, JsonValue]) -> bool: + """Prove an encrypted compaction replacement without making it portable.""" + + if payload.get("conversation") not in (None, "") or payload.get("previous_response_id") not in (None, ""): + return False + input_value = payload.get("input") + if not isinstance(input_value, list) or not input_value: + return False + input_items = cast(list[JsonValue], input_value) + compaction_item_value = input_items[0] + if not isinstance(compaction_item_value, dict): + return False + compaction_item = cast(dict[str, JsonValue], compaction_item_value) + if ( + set(compaction_item) != _SAME_ACCOUNT_COMPACTION_ITEM_FIELDS + or compaction_item.get("type") != "compaction" + or not _is_nonblank_string(compaction_item.get("id")) + or not _is_nonblank_string(compaction_item.get("encrypted_content")) + ): + return False + + suffix_payload: dict[str, JsonValue] = dict(payload) + suffix_payload["input"] = input_items[1:] + return responses_payload_is_account_neutral_fresh_replay(suffix_payload) + + def _reasoning_config_is_account_neutral(reasoning: JsonValue | None) -> bool: if reasoning is None: return True diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/design.md b/openspec/changes/recover-codex-desktop-idle-bridge/design.md index 45f132d300..b068059d5c 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/design.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/design.md @@ -4,7 +4,19 @@ The HTTP Responses bridge serializes upstream `response.create` submissions with The production request that motivated this change was eventless after its current `response.create` send and remained pending for 3,467 seconds. Codex Desktop disconnected first because its parsed-event idle timeout is 300 seconds. The backend route had classified the native request as OpenAI-SDK-shaped from its payload and `Accept` header, so periodic SSE comments never reached the parsed-event timer. -Current `main` already provides terminal request settlement, whole-session retirement, a stuck-retirement Prometheus counter, lifecycle locking, native Codex identity detection, and safe later-waiter recovery. This change reuses those primitives directly and does not import PR #1394's retry-circuit, replay, coordinator, or migration surface. +After the owner-side watchdog shipped, the same long-running session exposed a durable recovery loop. Each client retry created a fresh upstream socket, but durable reattach injected the last completed response id into the full-context resend. The new socket again produced no lifecycle event, the watchdog retired it after 240 seconds, and durable release left the same anchor available for the next retry. + +After quarantine was added, the observed Codex CLI retry still failed closed. Its stored 71-item prefix matched the durable fingerprint exactly, while the projected suffix contained assistant commentary, a complete `custom_tool_call` / `custom_tool_call_output` pair, and later developer/user messages. The generic full-resend predicate requires a later completed assistant message followed by fresh user input unless a durable pending-call manifest exists. Quarantine intentionally clears that manifest, so the valid mid-tool full-history resend was misclassified as incremental. + +Production aggregation later found ten client-visible disconnect failures across seven conversations and three upstream accounts. In the later five conversations, a `stream_incomplete` on one upstream socket was immediately followed by one `missing_response_created_timeout` on the same conversation and account. Each active conversation later succeeded and none repeated the missing-created timeout, which shows that quarantine stops the loop but currently acts one request too late. The local process did not restart during those later incidents; observed upstream close modes included no close frame, 1000, 1001, and 1012. + +At 19:32:10 in a later incident, one shared environment-proxy EOF terminated seven Responses WebSockets across four accounts in 358 milliseconds. One owner account had three concurrent requests, so three ordinary `stream_incomplete` health writes crossed the load balancer's transient-error threshold even though its credentials, quota, and persisted status remained healthy. Continuity-bound retries then received `previous_response_owner_unavailable` until the local backoff expired. The request that first exposed the 502 arrived later and was not the trigger. + +A subsequent Goal-enabled session showed that the local fail-closed contract could itself create a retry storm. After one real WebSocket failure quarantined the old anchor, roughly 57 automatic incremental continuations were rejected by `quarantined_anchor_requires_full_resend`. Those rejections created no upstream transport, yet each reused the retryable `stream_incomplete` 502 and the message "Upstream websocket closed before response.completed". Goal interpreted the response as another transient disconnect and kept retrying. A fresh Goal session worked, so the repeated failures required a client action—complete context or a new session—not another automatic retry. + +The upstream protocol explains the sequence. OpenAI documents that a Responses WebSocket handles one in-flight response, currently tops out at 60 minutes, and keeps the most recent response warm in a connection-local cache. Its reconnect guidance requires `store=false` or otherwise unresolvable chains to restart with `previous_response_id` omitted and full input context. Re-injecting a durable id from the closed socket into a fresh socket therefore cannot be treated as durable continuity. + +Current `main` already provides terminal request settlement, whole-session retirement, a stuck-retirement Prometheus counter, lifecycle locking, native Codex identity detection, durable continuity coordination, and safe later-waiter recovery. This change reuses those primitives directly and does not import PR #1394's retry circuit, replay policy, new schema, or migration surface. ## Goals / Non-Goals @@ -13,14 +25,27 @@ Current `main` already provides terminal request settlement, whole-session retir - Terminate an eventless request that remains pre-`response.created` before the native client's 300-second idle boundary, without requiring another gate waiter. - Measure the deadline from the current upstream send rather than request construction or admission wait. - Fail closed through existing settlement and session-retirement paths without replaying ambiguous work or moving it to another account. +- Prevent a proxy-injected durable anchor that reaches the eventless deadline from being re-injected indefinitely, without erasing a newer concurrent anchor. +- Preserve proxy-injected anchor provenance when the request is forwarded to another bridge owner. +- Reject a post-quarantine incremental request instead of silently submitting it as a fresh turn. +- Recover a fingerprint-matched Codex full-history resend that advances through a complete, self-contained direct tool-call/output round trip, even before a later assistant-final or user message exists. +- Invalidate a sent proxy-injected `store=false` anchor as soon as its upstream socket is known closed, before any existing safe no-anchor replay can mutate the request state, so a later full-history retry does not spend another 240 seconds on the dead connection-local id. +- Keep quarantined durable rows fail-closed even when historical input proof is absent, and retain injected-anchor provenance through a same-anchor local rebind. +- Prevent one process-local, shared-egress Responses WebSocket EOF from being counted as independent account failures while preserving ordinary one-account circuit-breaker signals. +- Stop deterministic full-resend-required guards from masquerading as retryable upstream disconnects. - Send parser-visible liveness to verified native Codex clients while retaining OpenAI event normalization when their payload needs it. - Preserve structured low-cardinality retirement metrics and logs. **Non-Goals:** - Replay a pre-visible request, retry clean upstream closes, or persist retry cooldowns across replicas. +- Hide or transparently replay the request interrupted by the upstream disconnect after response lifecycle or downstream-visible evidence; without an upstream idempotency/resume contract, that could duplicate model or tool side effects. - Recover a request after any matched `response.*` lifecycle event; eventful missing-created recovery remains outside this narrow change. - Change account selection, continuity ownership, request budgets, public `/v1/responses`, or operator settings. +- Replay the timed-out request without its anchor or turn an incremental continuation into an automatic fresh turn; anchorless recovery is left to a later self-contained client resend. +- Relax generic or cross-account replay rules, accept account-scoped/unsupported state in the newly appended suffix, move an owner-bound retained prefix to another account, or infer safety from an orphaned or incomplete tool item. +- Correlate disconnect incidents across replicas, suppress failures with complete close frames, or make post-dispatch transport failures replayable. +- Change owner-unavailable, ordinary network, raw previous-response lookup, or other potentially recoverable continuity errors from their existing retryable contracts. - Merge PR #1394, deploy the result, or alter the current Mac mini runtime as part of this code change. ## Decisions @@ -63,18 +88,134 @@ Continue using `_is_openai_sdk_request` to decide whether response events need t Explicit `x-stainless-*` headers or an OpenAI User-Agent retain comment liveness. Public `/v1/responses` never enables the native heartbeat override. This changes only liveness framing; it does not relax authentication, routing, payload validation, fingerprint normalization, or vendor-event filtering. +### 6. Quarantine only the exact proxy-injected durable anchor + +If the expired eventless owner used a proxy-injected `previous_response_id`, clear the durable row's `latest_response_id` and pending-tool metadata before durable session release. Retain `latest_input_item_count` and `latest_input_full_fingerprint` as the proof needed to distinguish a safe full-context resend from an incremental request. The update is one compare-and-set statement fenced by `(session_id, owner_instance_id, owner_epoch, expected_latest_response_id)`. + +The response-id alias remains available for explicit client continuity and owner resolution; quarantine removes only the automatic latest-anchor injection state. A client-supplied `previous_response_id` is never quarantined by this watchdog. + +If the durable owner or epoch changed, or another response advanced `latest_response_id`, the update mutates nothing. The newer owner or anchor remains authoritative. A persistence error or the bounded five-second persistence timeout is logged, but it does not prevent terminal settlement of the already ambiguous in-memory session. + +The timed-out request is not replayed after quarantine. Durable lookup derives quarantine from `latest_response_id is None` together with a non-null input count and fingerprint; no redundant schema field is added. A later request without an explicit anchor may establish a new upstream response lineage only when the existing prefix/fingerprint and safe-full-resend checks prove that its payload contains the prior output and a fresh follow-up. An incremental, mismatched, or otherwise unverifiable request fails closed with the existing retryable continuity error. + +### 7. Bind proxy-injected provenance across owner forwarding + +Add a boolean proxy-injected-anchor field to `HTTPBridgeForwardContext` and its reserved internal header. Include it in the canonical structured HMAC payload. When the field is true, the origin uses the versioned structured primary signature as well as the full-body signature, and the receiver does not accept a downgrade to the delimiter-based legacy primary signature. + +This makes stripping, adding, or changing the provenance marker fail signature validation. During a mixed-version rollout, an updated origin forwarding a proxy-injected anchor to an owner that cannot authenticate the new context fails closed rather than silently losing quarantine provenance. Requests without proxy-injected provenance retain the existing rolling-upgrade compatibility path. + +### 8. Recognize self-contained mid-tool history only during quarantine recovery + +Keep the existing completed-assistant-plus-new-user and durable pending-call-manifest checks unchanged. Add one quarantine-only alternative after the durable input count and raw prefix fingerprint have already matched. The projected entire input must have a self-contained call/output graph. The projected suffix after the stored boundary must independently satisfy the strict account-neutral fresh-input validator and contain at least one complete supported direct tool-call/output pair. A later assistant-final message or new user message is not required for this alternative because the paired output itself is the fresh continuation evidence. + +This path validates a later client-generated full-history resend; it does not replay the request that timed out. It does not change account selection or enable cross-account replay. The fingerprint-proven retained prefix may contain existing owner-bound `additional_tools` declarations because it remains on the durable owner account; it is not required to become cross-account portable. Requiring a self-contained whole-history call graph plus independently account-neutral suffix rejects an output that depends on a call before the stored boundary, duplicate call ids, unsupported or account-scoped suffix items, orphan outputs, and calls without outputs. Requiring an actual pair prevents ordinary incremental messages from using this alternative. + +### 9. Make quarantine fail closed without historical prefix proof + +Some historical durable anchors do not have both a positive input count and a non-empty fingerprint. Clearing such an anchor to null values would make the row indistinguishable from a genuinely fresh session, allowing a later incremental request to start a new lineage without prior context. + +The same fenced compare-and-set that clears the exact response id writes a reserved negative input count and deterministic non-empty fingerprint only when usable positive proof is absent or incomplete. Existing positive proof remains unchanged. Durable lookup therefore still derives quarantine from the existing columns without a schema flag, while `_input_prefix_matches_stored_context` rejects the negative count unconditionally. A later normally completed response overwrites the sentinel with real input proof. + +A local request state reconstructed after an owner-forward failure also retains proxy-injected provenance only when its `previous_response_id` is exactly the same as the original state. A changed or removed id does not inherit provenance. + +### 10. Quarantine connection-local anchors at socket disconnect + +Record the validated Responses `store` boolean on each request state. Before attempting the existing pre-created replay after a non-text upstream disconnect, snapshot an eligible anchor candidate under the pending-request lock. Eligibility requires an HTTP request that was actually sent, is not draining, has `store=false`, and carries a proxy-injected non-null `previous_response_id`. Queued requests have no current send timestamp and are excluded. + +Also record the effective `store` value for the latest response completed on the current socket. Clear that current-socket provenance whenever the upstream is replaced. Prefer the unique current response-create gate owner, then an eligible anchor matching the session's last completed response id, then a single distinct eligible sent anchor. If no sent candidate exists but the exact latest response is proven to have completed with `store=false` on the disconnecting socket, select that id so an idle close is covered. If multiple different sent anchors remain ambiguous and no current-socket latest id is proven, do nothing. + +Snapshot the immutable response id and quarantine it before the existing safe no-anchor replay can clear provenance from mutable request state. When the compare-and-set confirms the exact durable anchor was cleared, clear the same in-memory latest-response id, its current-socket provenance, and pending-tool metadata while retaining input count and fingerprint; this prevents a replay-retained local session from re-injecting the old id. A CAS miss, persistence failure, fenced owner, or newer response leaves in-memory continuity untouched. The replay remains allowed and a later completed response replaces the quarantined state. The owner/epoch/expected-response compare-and-set remains the final authority, so a stale candidate cannot erase a newer anchor. + +This invalidates only the automatic reattach optimization. It does not create a new replay path, move accounts, add an account-health write, clear historical aliases, or alter the client-visible `stream_incomplete` for work whose upstream acceptance or side effects are ambiguous. An already-proven safe full-context replay may still run without the anchor. Otherwise Codex's subsequent full-history retry reaches the existing quarantine guard immediately instead of first sending a dead connection-local id and waiting for the eventless watchdog. + +### 11. Treat every fresh store-false socket as a new lineage boundary + +An idle-close handler cannot run after a process crash or restart. Durable lookup may therefore find a non-null latest response id whose connection-local cache disappeared with the old process. Do not copy or inject that id into a fresh WebSocket merely because the durable row still owns account and prefix metadata. + +Before and after local session resolution, distinguish a forwardable live owner and an exact response completed with `store=false` on the resolved session's current socket from a true fresh lineage. A pre-resolution observation that some local session is live is not sufficient: a recovery socket can exist before its first response completes, and a concurrent incremental request must not use that bare socket as continuity proof. For a hard-continuity automatic-anchor request without either authoritative live path, apply the same retained-prefix and safe-full-resend predicates used by quarantine recovery, including the self-contained complete tool-call/output alternative. A verified full-history request remains unanchored. Incremental, mismatched, or unsupported input fails closed before transport creation or submission. Reapply this quarantine admission whenever an owner-forward failure refreshes and replaces the durable lookup, because the anchor may have been quarantined concurrently. These checks apply only to automatic `store=false` reattach; explicit client anchors and explicit `conversation` continuity keep their existing resolution paths. + +The durable id remains useful for account routing and prefix proof until a new response completes and overwrites it. Avoid a pre-claim mutation because another live owner or concurrently advanced response may still be authoritative. The absence of current-socket provenance also prevents the newly created session from copying that durable id into its in-memory automatic-anchor slot. + +### 12. Revalidate connection-local provenance at the final send boundary + +Session resolution is not the final submission boundary. A request can serialize a proxy-injected anchor, then wait behind the response-create gate while the reader observes a disconnect, quarantines the anchor, and replaces the socket. Gate waiters are not yet present in `pending_requests`, so disconnect selection cannot use their mutable request state as evidence. + +After gate acquisition and any closed-session recovery, revalidate the request's injected id against the session's latest response id and `store=false` provenance while holding the lifecycle lock that also covers enqueue and send. If the match still holds, preserve the existing anchored path. If it does not, use the captured unanchored request only when the existing replay-safety flag already proves that request is a self-contained full-history fallback; otherwise return the existing retryable continuity failure before appending or sending the request. When external-image inlining is enabled, prepare both serialized candidates before this late choice: inline and validate image URLs independently and apply the upstream size budget to each transformed frame. Preparing only the anchored frame would let a socket change restore raw external URLs from the stale captured fallback. When durable lookup replaces a different in-memory latest response id, clear the old id's socket-local `store` provenance together with pending-tool metadata so that provenance cannot bind the replacement id. + +### 13. Correlate ambiguous Responses receive failures before health settlement + +Treat only a typed Responses WebSocket receive failure without a complete peer close frame as a correlation candidate. Record the candidate under a credential-safe concrete-egress key: the actual routed proxy endpoint id, the parsed environment-proxy endpoint, or the direct destination. Do not derive identity or classification from exception message text. + +Before returning the adapter message to relay settlement, wait for at most one second. Two distinct non-empty upstream account ids on the same egress within that window establish a process-local correlated outage. Notify all waiters from that incident before any can reach account-health settlement, retain a short bounded observation history so trailing candidates in the same window receive the same result, and classify them as `proxy_network_unavailable`. Keep the observation store bounded and cancellation-safe. + +The direct Responses relay keeps one owned upstream receive task across downstream keepalive ticks. A keepalive timeout may emit liveness, but it does not cancel and restart an adapter receive that is already making its bounded correlation decision. Request-budget, stream-idle, and eventless-response deadlines likewise defer only when that owned receive task has already entered the bounded classification; its completed transport result reaches settlement before a local timeout can overwrite it. A truly silent receive remains subject to the original deadline. Relay shutdown and terminal settlement still cancel and await that owned task. + +Repeated failures from one account, failures on different concrete egresses, anonymous accounts, explicit close frames, and live sideband sockets retain their existing classification. A correlated failure is account neutral but remains post-dispatch and therefore is not replay-safe, does not move continuity ownership, and does not switch accounts. Process-local scope deliberately matches the observed single-replica incident without adding distributed state or a schema migration. + +### 14. Make deterministic full-resend admission failures client-actionable + +Use one dedicated HTTP bridge error for guards that have already proved an incremental continuation cannot recover on the current lineage: a quarantined automatic anchor without a verified full resend, a retained connection-local anchor on a fresh socket without a verified full resend, or a gate waiter whose proxy-injected anchor lost current-socket provenance and has no already-proven safe unanchored fallback. + +Return HTTP 400 with `error.code = "continuity_requires_full_resend"`, `error.type = "invalid_request_error"`, and `error.param = "input"`. The stable message instructs the client to resend the complete conversation context in `input` or create a new session. It does not claim that an upstream WebSocket just closed. Repeating the same incremental Goal request returns the same deterministic error before transport creation and cannot write account health. + +Keep the helper narrow and call it only from those guards. Owner lookup failures, active-owner unavailability, network failures, upstream close settlement, raw previous-response recovery, and general continuity loss retain their existing retryable errors because a later identical attempt may recover without changing the request. + +### 15. Quarantine the socket lineage before downstream-cancellation retirement + +Pressing Escape closes the downstream SSE generator before the upstream response reaches a terminal event. The detach path must continue to mark the request draining and retire the whole shared socket, because anonymous late frames from the abandoned response cannot safely share a socket with a later request. That local retirement cancels the upstream reader before calling the WebSocket close operation, so it cannot rely on the reader's non-text disconnect branch to quarantine connection-local state. + +While the pending lock still exposes pre-drain request provenance, snapshot the same exact anchor candidate used by disconnect invalidation. After detaching the downstream consumer but before whole-session close releases durable ownership, apply the existing owner/epoch/expected-response compare-and-set. On confirmed quarantine, clear only the matching in-memory latest response id, current-socket `store` provenance, and pending-tool metadata; retain the input count and fingerprint for the next full-history recovery. A missing candidate, `store=true`, client-supplied anchor, unsent request, ambiguous candidates, CAS miss, persistence failure, or newer response keeps the existing fail-closed behavior. + +This is lineage invalidation, not replay. The interrupted request is never resubmitted, the old socket is never reused, the account does not change, and cancellation does not write account health. The next client turn must still satisfy the existing verified full-history predicate before it can start unanchored on a new socket. + +### 16. Recognize upstream encrypted compaction as same-account complete context + +Codex automatic compaction replaces the prior client-visible item history with an upstream-issued opaque item shaped exactly as `{"id": "...", "type": "compaction", "encrypted_content": "..."}`. Its ciphertext cannot and should not match the durable fingerprint of the plaintext history it replaces. Treat that representation as an independent fresh-socket admission proof only for a hard-continuity durable lookup that retains a positive input count, a non-empty fingerprint, and a concrete owner account. + +The classifier requires the compaction item to be first, requires non-blank `id` and `encrypted_content`, rejects unknown fields, and runs all later input through the existing account-neutral self-contained replay validator. The request must omit explicit `previous_response_id` and `conversation`. A valid request opens or takes over a fresh socket without injecting the old connection-local response id and forwards the compaction item byte-for-byte while keeping selection fixed to the durable owner. The ciphertext is never projected, stripped, or made eligible for account-neutral failover. + +This exception does not authenticate arbitrary ciphertext locally; upstream remains authoritative for its encrypted format. Its safety boundary is narrower: a malformed or forged item can affect only the caller's same durable session on the same upstream account and will be rejected by upstream, while a missing id, blank ciphertext, extra field, plaintext summary, account-scoped suffix, absent durable proof, missing owner, soft-affinity request, or ordinary incremental input remains fail-closed before transport creation. + ## Risks / Trade-offs - **A send fails after the timestamp is set.** Existing send-error cleanup retires or settles the request before the watchdog can act; tests cover that the timestamp alone is not sufficient eligibility. - **A quiet upstream accepted the request but emitted no event.** The proxy returns an explicit failure rather than risking a duplicate replay. The selected account remains healthy because silence is not proof of account failure. - **A matched lifecycle event arrives just before timeout.** Eligibility is rechecked under the existing request/session synchronization before retirement, and any matched `response.*` event suppresses this watchdog. +- **A newer response commits while quarantine is pending.** The conditional response-id predicate preserves the newer anchor and all metadata coupled to it. +- **Durable ownership changes while quarantine is pending.** Owner and epoch fencing makes the stale watchdog write a no-op; the replacement owner remains authoritative. +- **A later request contains only incremental input.** Retained input proof marks the durable row as quarantined, and the request fails before transport creation instead of becoming a fresh turn. +- **A later request stops at a normal tool boundary.** A fingerprint-matched, fully projected history with a complete direct call/output suffix may start a new lineage without waiting for a synthetic user turn. +- **A malformed tool suffix resembles a full resend.** Whole-history and suffix self-containment plus the complete-pair requirement reject orphan outputs, unresolved calls, duplicate ids, and unsupported tool state. +- **A plaintext summary resembles automatic compaction.** Only the exact upstream encrypted-compaction item shape is admitted, and the request remains pinned to the durable owner account; ordinary messages and cross-account replay remain ineligible. +- **A historical anchor has no usable input proof.** The quarantine sentinel remains identifiable but can never satisfy the prefix matcher, so recovery fails closed until a real completed response replaces it. +- **A queued request happens to name the same anchor when a socket closes.** Absence of a current send timestamp excludes it from disconnect invalidation. +- **The socket closes while completely idle.** Current-socket completion provenance permits an exact latest-response quarantine even with an empty pending queue. +- **A process exits without observing the close.** The next fresh socket treats the retained automatic `store=false` id as recovery proof only and never injects it. +- **The socket changes while an anchored request waits for the gate.** Final send-boundary revalidation either removes the stale anchor from an already-proven full-history request or fails the dependent incremental request before upstream submission. +- **The safe unanchored fallback contains an external image.** Both candidates are inlined, validated, and size-checked before final selection, so choosing the fallback cannot restore an upstream-incompatible URL or bypass the frame budget. +- **A durable id is loaded into a new process but its origin is unknown.** Unknown current-socket provenance prevents idle-close inference; the fresh-socket full-context guard, not an unfenced guess, controls recovery. +- **Two accounts independently lose no-close sockets close together.** They may be classified as a shared egress incident, but neither transport failure is credential evidence; keeping both accounts selectable is safer than poisoning continuity owners, and the interrupted requests still fail without replay. +- **One broken account repeatedly drops sockets.** Distinct-account counting prevents its concurrent requests from manufacturing a shared incident, so the existing transient health penalty remains active. +- **Two different proxy endpoints fail together.** Concrete-egress keys keep their observations separate, so each retains ordinary account-health treatment unless its own endpoint has cross-account evidence. +- **The process shuts down during the correlation wait.** Task cancellation interrupts the bounded wait, and stale observations expire from a bounded in-memory store without owning transport resources. +- **A local deadline expires during correlation.** The already-observed transport failure completes its bounded classification first; silent reads that never entered correlation still settle at the normal deadline. +- **A Goal client does not understand the new code.** HTTP 400 stops generic transient retry behavior, while the message still gives two safe recovery actions; the service never converts the incremental request into a fabricated fresh turn. +- **A transient continuity failure resembles the deterministic guard.** Only the three proven full-resend boundaries use the new helper; all other continuity paths retain their current retryable status. +- **Several sent requests name different anchors.** The selector acts only with a unique gate owner, an exact session-latest match, or one distinct candidate; otherwise the fenced durable state is left unchanged. +- **Durable quarantine succeeds while safe replay keeps the local session.** The matching in-memory latest anchor and pending-tool metadata are cleared so the local session cannot undo durable quarantine; input proof remains available. +- **The disconnect CAS misses or persistence fails.** The proxy does not claim in-memory quarantine succeeded and leaves local continuity fields unchanged; normal failure/retry settlement remains authoritative. +- **A persisted response id remains resolvable after reconnect.** Proactive disconnect invalidation requires `store=false`; `store=true` requests retain the existing reconnect behavior. +- **The disconnect interrupted already-visible work.** The proxy still fails that request instead of replaying it; only the dead automatic anchor is quarantined for the client's next self-contained retry. +- **The downstream client cancels before the upstream peer closes.** Detach snapshots and quarantines the same exact current-socket anchor before canceling/releasing the bridge, then preserves the ownership barrier and existing full-history admission for the next turn. +- **A forwarding hop tampers with or strips provenance.** Versioned structured signatures bind the marker and prevent fallback to an unbound legacy signature. +- **A mixed-version owner cannot verify the new provenance field.** That cross-replica request fails closed during rollout; non-provenance forwards remain backward compatible. - **Whole-session retirement interrupts a healthy sibling.** This narrow design chooses fail-closed session cleanup rather than attempting unsafe sibling isolation on current `main`. Existing terminal settlement must cover every pending sibling exactly once. - **A client spoofs native identity.** The only benefit is an ignored vendor liveness event on the authenticated Codex backend route; explicit SDK markers still take precedence. ## Migration Plan -No data migration or setting change is required. Deploying a new process initializes the monotonic field on new in-memory request states. Rollback restores the previous timeout and heartbeat selection without persistent-state conversion. +No data migration or setting change is required. Deploying a new process initializes the monotonic field and bounded correlation state in memory and uses existing nullable durable-anchor columns. Rollback restores the previous timeout, durable-anchor retention, heartbeat selection, and per-disconnect health classification without persistent-state conversion. ## Open Questions -None. The scope intentionally solves only the observed eventless/no-waiter production failure. +None. The scope intentionally removes the observed eventless/no-waiter wedge, the immediately preceding connection-local-anchor reinjection window, and the observed shared-egress health amplification without introducing transparent replay. diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md b/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md index d9849d4af5..a9ada5c19f 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/proposal.md @@ -2,13 +2,43 @@ A production Codex Desktop request on the HTTP-to-WebSocket bridge remained pending for nearly an hour after its upstream `response.create` send produced neither `response.created` nor any matched `response.*` lifecycle event. Existing stuck-gate recovery runs only when another request later times out waiting for the gate, so a lone request can outlive the native client's 300-second parsed-event idle timeout. During the same wedge, the backend route classified the native Desktop request as an OpenAI SDK stream from its payload shape and emitted SSE comments that the Codex parser does not observe. +Production follow-up evidence exposed a second recovery gap: after the eventless watchdog retires the socket, the durable session still retains the proxy-injected `latest_response_id`. A later full-context client resend creates a fresh socket, receives the same automatic anchor, and can repeat the same 240-second eventless failure indefinitely. + +Adversarial review exposed two related safety gaps in that quarantine path. Cross-replica owner forwarding did not carry whether `previous_response_id` was proxy-injected, so the receiving owner could mistake it for a client-supplied anchor and skip quarantine. The initial quarantine mutation also erased the durable input count and fingerprint, so a later incremental request without an explicit anchor could be submitted as a fresh turn and silently lose prior context. + +Production validation then exposed an overly narrow recovery predicate. Codex CLI retries an HTTP stream by rebuilding the full local session history, including turns that stop at a completed `custom_tool_call` / `custom_tool_call_output` pair. Such a mid-tool resend may be fully self-contained and may not yet contain a later assistant-final message or new user message. Treating every such resend as incremental leaves a quarantined session in a repeated retryable-502 loop even when its retained prefix fingerprint matches exactly. + +Fleet-wide follow-up evidence exposed an earlier invalidation point. Across seven Codex conversations and three upstream accounts, each conversation recorded one upstream `stream_incomplete` followed by one approximately 240-second `missing_response_created_timeout`; active conversations later recovered and did not repeat the timeout after quarantine. OpenAI's WebSocket guidance states that connections currently end after at most 60 minutes, the most recent response is cached on that connection, and a disconnected `store=false` chain must restart with `previous_response_id` omitted plus full input context. The remaining timeout therefore comes from re-injecting a connection-local durable anchor after its socket has already closed, before the existing eventless watchdog has a chance to quarantine it. + +Final review found two variants without a sent pending request to identify the dead anchor. A socket may close while completely idle after its latest response completed, and a process restart loses the in-memory socket before any close handler can run. The former still has current-socket completion provenance and can use the same fenced quarantine. The latter must treat a retained `store=false` durable id as routing and recovery proof only: a fresh socket may accept a fingerprint-verified full-history resend without the id, but it must fail an incremental or unverifiable request closed instead of injecting the connection-local id. + +A later production incident exposed an account-health amplification path around those disconnects. One shared environment-proxy EOF terminated seven Responses WebSockets across four accounts within 358 milliseconds. Each ordinary `stream_incomplete` was counted independently, so the third concurrent failure on one still-valid owner account crossed the transient-error threshold and made every continuity-bound retry fail with `previous_response_owner_unavailable`. The account remained active, had valid credentials and quota, and became selectable again after the local backoff. A bounded cross-account correlation decision is therefore required before ambiguous no-close receive failures reach account-health settlement. + +The same production follow-up exposed a client-actionability bug after quarantine worked as designed. One Goal-enabled Codex session saw a single real transport failure, then automatically submitted roughly 57 incremental continuations against the quarantined lineage. Every continuation was rejected locally with `quarantined_anchor_requires_full_resend`, but the bridge returned the generic retryable `stream_incomplete` 502 and falsely said that the upstream WebSocket had just closed. Goal therefore retried a request that could recover only if the client resent complete context or created a new session. A new Goal-enabled session worked normally, confirming that these later failures were deterministic request-state rejections rather than repeated transport outages. + +A later Codex CLI reproduction exposed one remaining disconnect variant. Pressing Escape closes the downstream SSE while the upstream response is still running. The bridge correctly treats that cancellation as an ownership barrier and retires the shared upstream socket, but its close path cancels the upstream reader before a peer-close event can reach the existing disconnect quarantine. The connection-local `store=false` anchor therefore remains durable even though its socket has been intentionally destroyed, and later turns can become trapped in reconnect/full-resend failures until the user creates a new Codex session. + +A production session then exposed a distinct recovery representation after that lineage was repaired. Codex automatically compacted a successful 215k-token turn and replaced the previously fingerprinted full input history with one upstream-issued encrypted `compaction` item. The next automatic continuation reached a fresh socket, but the durable guard compared that opaque replacement against the pre-compaction prefix fingerprint and rejected it as `continuity_requires_full_resend`. An encrypted compaction is complete same-account context by protocol, not an incremental plaintext summary, so the guard needs a narrow account-pinned alternative without weakening ordinary full-resend or cross-account replay checks. + ## What Changes - Record the monotonic time of the current upstream `response.create` send. - Proactively expire an eventless request that remains pre-`response.created` for the smaller of the existing stuck-gate threshold and 240 seconds, even when no second gate waiter exists and periodic keepalives are disabled. - Fail the affected bridge session closed through existing terminal settlement and retirement paths, without transparent replay, account movement, or account-health penalties. +- When the timed-out request used a proxy-injected durable anchor, conditionally quarantine that exact anchor and pending-tool metadata using the current durable owner and epoch while retaining the input count and fingerprint as proof that recovery is required. +- Preserve proxy-injected anchor provenance across owner forwarding in the structured HMAC context, reject tampered or downgraded provenance, and apply the same quarantine behavior on the receiving replica. +- Allow a quarantined session to recover without an anchor only from a fingerprint-verified safe full-context resend, including a self-contained mid-tool suffix with a complete direct tool-call/output pair; fail incremental, orphaned-tool, incomplete-tool, or otherwise unverifiable requests closed with the existing retryable continuity error. +- Preserve a fail-closed quarantine marker when an older durable anchor has no usable positive input count and fingerprint, and preserve proxy-injected provenance when an owner-forward failure re-prepares the same anchored request locally. +- As soon as an upstream socket disconnects, proactively quarantine the exact connection-local anchor used by a sent `store=false` HTTP bridge request before running the existing safe no-anchor replay. The selector ignores queued, client-supplied, persisted, or ambiguous anchors, and the existing fenced compare-and-set preserves newer durable progress. +- Record whether the session's latest completed response belongs to the current socket so an idle disconnect can quarantine that exact `store=false` anchor even when no request remains pending, and clear that provenance whenever the upstream socket changes. +- On a fresh socket with no live local or forwardable owner, never inject a retained automatic `store=false` durable id. Allow only the same fingerprint-verified full-context recovery accepted after quarantine; reject incremental or unverifiable input before creating an upstream transport. +- Revalidate every proxy-injected connection-local anchor after response-create admission waits and immediately before send. If the socket lineage changed, use an already-proven safe full-history fallback without the anchor or fail continuity closed before submission. +- Return a stable non-retryable HTTP 400 `continuity_requires_full_resend` invalid-request error for deterministic local guards that can recover only from complete context or a new session. Do not describe those guards as an upstream WebSocket close; keep genuinely transient owner, transport, and generic continuity failures on their existing retryable 5xx paths. +- Treat downstream SSE cancellation that retires the upstream bridge as the same connection-local lineage boundary as an observed upstream disconnect: select and quarantine the exact eligible `store=false` automatic anchor before durable ownership is released, then close the old socket without replaying the interrupted request. +- Admit an upstream-issued encrypted `compaction` item as a same-account complete-context replacement on hard-continuity fresh-socket or quarantined-anchor recovery. Require exact item shape, retained durable proof, a fixed durable owner, and account-neutral self-contained suffix content; never move the ciphertext across accounts or relax ordinary incremental admission. +- Hold ambiguous Responses WebSocket receive failures without a complete close frame for at most one second before account-health settlement. When at least two distinct accounts fail on the same concrete egress in that window, classify every correlated candidate as account-neutral `proxy_network_unavailable`; retain existing penalties for one-account, different-egress, and explicit-close failures. - Give verified native Codex identity parser-visible `codex.keepalive` frames even when payload-shape heuristics still require OpenAI-compatible event normalization; explicit SDK markers and public `/v1/responses` retain comment liveness. -- Add regressions for the no-waiter deadline, protected created/eventful requests, account-neutral retirement, and contrasting Desktop/SDK/public heartbeat contracts. +- Add regressions for the no-waiter deadline, protected created/eventful requests, account-neutral retirement, fenced anchor quarantine, cross-replica provenance, completed mid-tool recovery, incremental and malformed-tool fail-closed behavior, concurrent-anchor preservation, correlated egress failures, preserved single-account penalties, and contrasting Desktop/SDK/public heartbeat contracts. ## Capabilities @@ -18,12 +48,13 @@ None. ### Modified Capabilities -- `proxy-admission-control`: Add a proactive, fail-closed deadline for an eventless response-create gate owner without requiring another waiter. -- `responses-api-compat`: Require verified native Codex Desktop HTTP streams to receive parsed-event liveness frames without weakening SDK normalization. +- `proxy-admission-control`: Add a proactive, fail-closed deadline for an eventless response-create gate owner without requiring another waiter, and distinguish deterministic full-resend admission failures from retryable transport continuity failures. +- `responses-api-compat`: Require verified native Codex Desktop HTTP streams to receive parsed-event liveness frames without weakening SDK normalization, prevent an eventless proxy-injected durable anchor from being automatically re-injected forever, expose full-resend-required guards as actionable non-retryable client errors, and keep correlated shared-egress disconnects from poisoning account health. +- `outbound-http-clients`: Extend account-neutral network classification to bounded, same-egress, cross-account Responses WebSocket receive failures without complete close frames. ## Impact -- Affected code: HTTP bridge request send timing, upstream-reader timeout/retirement, backend Responses client identity, and SSE keepalive selection. -- Affected surfaces: `POST /backend-api/codex/responses` and its server-side upstream WebSocket bridge. -- No new setting, dependency, database migration, public endpoint, retry circuit, durable coordinator, or OpenAI SDK contract change. +- Affected code: HTTP bridge request send timing and current-socket `store` provenance, fresh-socket durable recovery admission and error mapping, same-account encrypted-compaction classification, upstream-reader timeout/disconnect retirement, downstream-cancellation retirement, signed owner forwarding, durable bridge continuity persistence, quarantine-only replay-safety classification, bounded WebSocket egress-failure correlation, backend Responses client identity, and SSE keepalive selection. +- Affected surfaces: `POST /backend-api/codex/responses`, direct Responses WebSocket relay, and their server-side upstream WebSocket adapters. +- No new setting, dependency, database migration, public endpoint, retry circuit, or OpenAI SDK contract change. The existing durable coordinator gains one fenced conditional quarantine operation over nullable columns. - The change is independent of PR #1394. It fixes the observed eventless/no-waiter wedge but intentionally does not add #1394's transparent replay, clean-close retry, or cross-replica cooldown behavior. diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/specs/outbound-http-clients/spec.md b/openspec/changes/recover-codex-desktop-idle-bridge/specs/outbound-http-clients/spec.md new file mode 100644 index 0000000000..76310153a3 --- /dev/null +++ b/openspec/changes/recover-codex-desktop-idle-bridge/specs/outbound-http-clients/spec.md @@ -0,0 +1,61 @@ +## MODIFIED Requirements + +### Requirement: Process-wide network failures are account neutral + +The proxy MUST NOT record a transient, permanent, quota, rate-limit, or circuit-breaker health failure against an account when an attempt fails because the local process cannot resolve or route to the upstream host. Routed proxy transport failures MUST retain a credential-safe machine-readable classification after the original exception message is sanitized. A permanent missing proxy hostname MUST remain an endpoint-scoped proxy failure rather than entering process-wide recovery. + +A Responses WebSocket receive failure without a complete peer close frame MUST also remain account neutral when a bounded process-local correlation window observes failures from at least two distinct non-empty upstream account ids on the same concrete egress within one second. Concrete egress identity MUST distinguish the actual routed proxy endpoint, parsed environment-proxy endpoint, and direct destination without using exception message text or exposing proxy credentials. Every candidate in the correlated window MUST be classified before account-health settlement as `proxy_network_unavailable`. Downstream keepalive scheduling MUST NOT cancel or restart a pending correlation decision. Once an owned receive task has entered bounded no-close correlation, a request-budget, stream-idle, or eventless-response deadline MUST NOT cancel or settle that receive failure before the correlation decision completes; the completed receive classification MUST reach the existing settlement path first. Repeated failures from one account, failures on different egresses, anonymous accounts, explicit close frames, and live sideband sockets MUST retain their existing classification. Correlation MUST NOT authorize replay of a post-dispatch request, move continuity ownership, or switch accounts. + +#### Scenario: Wi-Fi transition does not poison account health + +- **WHEN** an upstream attempt fails with a classified local DNS or host-route failure +- **THEN** the selected account's health counters and cooldown state are unchanged +- **AND** the selected account's circuit breaker is unchanged +- **AND** continuity ownership remains pinned to that account + +#### Scenario: Routed transient DNS failure remains account neutral after sanitization + +- **WHEN** an HTTP or WebSocket attempt through a resolved upstream proxy route fails with transient DNS or local route loss +- **THEN** the credential-safe routed error carries the process-network classification +- **AND** the selected account's health and circuit-breaker state are unchanged + +#### Scenario: Missing proxy hostname remains endpoint scoped + +- **WHEN** resolving a configured upstream proxy hostname fails with a permanent name-not-found result +- **THEN** the failure remains `upstream_unavailable` +- **AND** the proxy does not classify the host process as disconnected + +#### Scenario: Shared egress EOF does not poison account health + +- **GIVEN** two Responses WebSockets for distinct upstream accounts use the same concrete egress +- **WHEN** both receive paths fail without complete peer close frames within one second +- **THEN** every correlated failure carries `proxy_network_unavailable` +- **AND** neither account receives a transient health or circuit-breaker failure +- **AND** no interrupted post-dispatch request is replayed or moved to another account + +#### Scenario: Downstream keepalive does not restart correlation + +- **GIVEN** the configured downstream keepalive interval is shorter than the no-close correlation window +- **WHEN** a Responses receive failure is waiting for its bounded correlation decision +- **THEN** keepalive scheduling does not cancel or restart that receive decision +- **AND** the failure reaches exactly one existing settlement path after correlation completes + +#### Scenario: Request deadline does not preempt in-flight correlation + +- **GIVEN** a Responses receive task has observed a no-close failure and entered bounded correlation +- **WHEN** the owning request budget, stream-idle window, or eventless-response deadline expires before the correlation decision completes +- **THEN** the receive task completes its bounded classification before settlement +- **AND** a correlated `proxy_network_unavailable` result remains account neutral instead of being replaced by a timeout health outcome + +#### Scenario: Single-account and different-egress failures remain account specific + +- **WHEN** no-close receive failures repeat only for one upstream account +- **OR** distinct accounts fail through different concrete egresses +- **THEN** the correlation threshold is not satisfied +- **AND** existing `stream_incomplete` account-health behavior remains authoritative + +#### Scenario: Explicit close frames are not inferred to be a shared EOF + +- **WHEN** an upstream Responses or live sideband WebSocket supplies a close frame +- **THEN** bounded no-close correlation does not reclassify that close +- **AND** the existing close-code and account-health contracts remain authoritative diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md b/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md index 51bbccd853..10eb82d8b7 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/specs/proxy-admission-control/spec.md @@ -8,6 +8,24 @@ The owner-side watchdog MUST apply only while the request owns the response-crea When the owner-side deadline expires, the proxy MUST recheck eligibility, emit a structured low-cardinality log and the existing stuck-retirement Prometheus counter, terminally fail and settle every pending request exactly once, and retire the whole bridge session. It MUST NOT transparently replay the timed-out request, move it to another account, or write an account-health failure for the missing-created timeout. +If the expired owner used a proxy-injected `previous_response_id` and the bridge owns a durable session row, the proxy MUST conditionally clear that row's automatic latest-response anchor and pending-tool metadata before releasing durable ownership. It MUST retain the latest input count and input fingerprint as recovery proof. The quarantine MUST be one compare-and-set mutation conditioned on the same session id, owner instance, owner epoch, and expected latest response id. A changed owner, changed epoch, concurrently advanced response id, client-supplied anchor, or nonmatching durable anchor MUST remain unchanged. The quarantine write MUST be bounded to no more than five seconds; a persistence error or timeout MUST be logged and MUST NOT prevent terminal settlement or durable release. + +If the exact anchor has no usable positive input count and non-empty fingerprint, that same compare-and-set mutation MUST write a reserved negative count and deterministic non-empty fingerprint so the row remains detectably quarantined and cannot be mistaken for fresh continuity. Existing usable proof MUST remain unchanged, the reserved proof MUST never satisfy a prefix match, and a later normal completed response MUST replace it with real input proof. + +Proxy-injected anchor provenance MUST survive cross-replica owner forwarding. The forwarding context and reserved header MUST carry the provenance boolean, the canonical structured HMAC payload MUST bind it, and a forward that claims proxy-injected provenance MUST NOT fall back to a legacy signature that does not bind the field. Adding, stripping, or changing the marker MUST either preserve a valid structured signature or reject the forwarded request. + +When an owner-forward attempt fails before yielding and the request is re-prepared for a local session, proxy-injected provenance MUST survive only when the re-prepared `previous_response_id` is exactly equal to the original id. + +When an upstream HTTP bridge WebSocket disconnects, or when downstream SSE cancellation forces the proxy to retire that upstream socket before a peer-close event can be observed, the proxy MUST proactively quarantine an exact connection-local anchor when either an actually sent request used that anchor with effective `store=false` and proxy-injected provenance or the session's latest completed response was produced with effective `store=false` on that same socket. The proxy MUST record current-socket `store` provenance when a response completes and MUST clear that provenance whenever the upstream socket changes. Sent-request candidates MUST be selected from non-draining HTTP requests under the pending lock, MUST prefer a unique current gate owner or an exact session-latest anchor, and MUST act on a single distinct anchor only when no stronger candidate exists. An exact current-socket latest response MAY be selected when no sent candidate exists. The proxy MUST snapshot the response id before marking a canceled request draining and MUST apply the quarantine before existing safe no-anchor replay or cancellation retirement can release durable ownership. When the compare-and-set confirms that exact durable anchor was cleared, the proxy MUST clear the same in-memory latest-response id, its current-socket provenance, and pending-tool metadata while retaining in-memory input count and fingerprint. A CAS miss, persistence failure, fenced owner, or newer durable response MUST NOT clear the in-memory continuity fields. The already-authorized safe replay MAY still run, and a later completed response MUST replace quarantine with its new anchor. Queued or unsent request anchors, `store=true` or unknown-provenance latest responses, client-supplied request anchors, and multiple ambiguous sent anchors MUST remain unchanged. The same owner/epoch/expected-response compare-and-set MUST protect concurrent durable progress. This invalidation MUST NOT replay an interrupted request, reuse its old socket, move accounts, or add an account-health write. + +When a hard-continuity request resolves a durable automatic latest-response id and supplies neither an explicit client `previous_response_id` nor `conversation`, an unanchored incremental request MAY proceed only through a forwardable live owner or a reusable local bridge session whose matching latest response completed with `store=false` on its current socket. The existence of a live local session without that socket-local completion MUST NOT authorize incremental submission. When no such live path exists, the proxy MUST treat the durable id as belonging to a previous socket and MUST NOT inject it. The proxy MAY submit the request unanchored only when the retained count and fingerprint plus the quarantine recovery predicates prove the supplied input is a self-contained full-history resend. Incremental, prefix-mismatched, or otherwise unverifiable input MUST fail closed with the full-resend-required error defined below before upstream transport creation or submission. A refreshed durable lookup after an owner-forward failure MUST reapply the same quarantine and full-resend admission before local takeover. A matching live current-socket response, a forwardable owner, an explicit client-supplied `previous_response_id`, and an explicit `conversation` remain governed by their existing paths. + +As a narrow alternative to plaintext prefix matching, the proxy MAY treat an upstream-issued encrypted `compaction` input item as complete context only for hard-continuity durable fresh-socket or quarantined-anchor recovery. The durable lookup MUST retain a positive input count, a non-empty fingerprint, and a concrete owner account; account selection MUST remain fixed to that owner and MUST NOT fall back or replay across accounts. The request MUST omit explicit `previous_response_id` and `conversation`. Its first input item MUST contain exactly `id`, `type`, and `encrypted_content`, with `type` equal to `compaction` and both other values non-blank strings. Every later input item and all remaining request controls MUST satisfy the existing account-neutral self-contained fresh-replay validator. An admitted request MUST be submitted unanchored with the compaction item preserved exactly. Missing or blank fields, unknown compaction fields, plaintext summaries, account-scoped or non-self-contained suffixes, missing durable proof or owner, soft-affinity requests, and ordinary incremental input MUST NOT use this alternative. + +Immediately after response-create gate acquisition and any closed-session recovery, the proxy MUST revalidate a proxy-injected anchor against the session's current-socket latest response id and `store=false` provenance while lifecycle ownership still excludes socket replacement. If the match was lost during admission, the proxy MUST NOT append or send the serialized anchored request. It MAY instead submit the captured unanchored request only when existing replay-safety proof marks that full request safe; otherwise it MUST fail closed with the full-resend-required error before submission. When HTTP bridge external-image inlining is enabled, both the anchored request and its captured unanchored fallback MUST undergo the same image inlining and surviving-URL validation before this final selection, and each transformed candidate MUST be checked against the upstream serialized request-size budget. When durable lookup replaces a different in-memory latest response id, the proxy MUST clear the previous id's current-socket `store` provenance and pending-tool metadata before the replacement id can influence automatic injection. + +When a quarantined automatic anchor, a retained automatic anchor on a fresh socket, or a final send-boundary lineage mismatch cannot pass the verified full-resend predicate, the proxy MUST return HTTP 400 with `error.code` equal to `continuity_requires_full_resend`, `error.type` equal to `invalid_request_error`, and `error.param` equal to `input`. The stable message MUST instruct the client to resend complete conversation context in `input` or create a new session and MUST NOT claim that an upstream WebSocket just closed. Repeating the same incremental request MUST produce the same local error without creating or submitting an upstream transport and without writing account health. Owner lookup, active-owner availability, transport, raw previous-response recovery, and other general continuity failures MUST retain their existing retryable contracts. + #### Scenario: Lone eventless gate owner is retired before the client timeout - **GIVEN** a visible HTTP bridge request owns the response-create gate @@ -45,3 +63,177 @@ When the owner-side deadline expires, the proxy MUST recheck eligibility, emit a - **THEN** every pending request is settled exactly once and the whole session is retired - **AND** the proxy does not replay the timed-out request or submit it on another account - **AND** the selected account is not marked unhealthy solely because `response.created` was missing + +#### Scenario: Eventless proxy-injected anchor is quarantined + +- **GIVEN** an eventless pre-created owner reaches the owner-side deadline +- **AND** its `previous_response_id` was injected by the proxy from the durable session's current latest-response anchor +- **WHEN** terminal retirement runs +- **THEN** the proxy clears that exact durable latest-response anchor and pending-tool metadata using owner-and-epoch fencing +- **AND** it retains the input count and fingerprint as quarantine recovery proof +- **AND** it does not replay the failed request without the anchor + +#### Scenario: Downstream cancellation quarantines before retiring the socket + +- **GIVEN** a downstream HTTP bridge stream is canceled after an eligible proxy-injected `store=false` anchor was sent on the current upstream socket +- **WHEN** detach retires that socket as an upstream ownership barrier +- **THEN** the exact anchor is conditionally quarantined before durable ownership is released +- **AND** confirmed matching in-memory anchor provenance is cleared while input proof is retained +- **AND** the interrupted request is not replayed, the old socket is not reused, and account health is not written + +#### Scenario: Automatic compaction starts a fresh same-account lineage + +- **GIVEN** a hard-continuity durable session retains prior input proof and a concrete owner account +- **AND** no reusable socket or forwardable owner can carry its connection-local automatic anchor +- **WHEN** the client sends an exact encrypted `compaction` item followed only by account-neutral self-contained input +- **THEN** the proxy opens a fresh socket on the durable owner account and omits the old `previous_response_id` +- **AND** it forwards the compaction item unchanged without enabling cross-account replay + +#### Scenario: Cross-replica owner preserves injected-anchor provenance + +- **GIVEN** an origin replica injects a durable `previous_response_id` +- **AND** forwards the request to the active owner replica +- **WHEN** the owner authenticates the internal forwarding context +- **THEN** it marks the owner-side request state as using a proxy-injected anchor +- **AND** an eventless deadline on that owner invokes the same fenced quarantine path + +#### Scenario: Forwarded provenance cannot be downgraded + +- **GIVEN** a signed owner forward carries proxy-injected anchor provenance +- **WHEN** the provenance header is added, stripped, or changed in transit +- **THEN** signature verification rejects the request +- **AND** verification does not fall back to a legacy signature that omits the provenance field + +#### Scenario: Quarantine persistence does not block settlement + +- **GIVEN** an eventless proxy-injected anchor reaches its deadline +- **AND** the durable quarantine write errors or exceeds five seconds +- **WHEN** terminal cleanup runs +- **THEN** the proxy logs the persistence outcome +- **AND** settles every pending request and releases durable ownership without waiting longer + +#### Scenario: Concurrent durable progress survives stale quarantine + +- **GIVEN** an eventless request was sent with proxy-injected anchor `resp_old` +- **AND** durable ownership changed or the durable latest-response anchor advanced after that send +- **WHEN** the stale request reaches the owner-side deadline +- **THEN** the conditional quarantine mutates no durable continuity fields +- **AND** the newer owner or response anchor remains available + +#### Scenario: Explicit client anchor is not quarantined + +- **GIVEN** an eventless pre-created owner carries a client-supplied `previous_response_id` +- **WHEN** the owner-side deadline expires +- **THEN** the proxy retires and settles the ambiguous bridge session +- **AND** it does not clear durable latest-response state solely because the explicit client anchor timed out + +#### Scenario: Missing historical proof remains fail closed after quarantine + +- **GIVEN** the exact proxy-injected anchor has no usable positive input count and fingerprint +- **WHEN** the fenced quarantine clears that anchor +- **THEN** the same mutation stores a reserved non-matching proof pair +- **AND** durable lookup continues to identify the row as quarantined +- **AND** no incoming prefix can match the reserved proof + +#### Scenario: Same-anchor local rebind preserves provenance + +- **GIVEN** an owner-forward request used a proxy-injected anchor +- **AND** forwarding fails before yielding +- **WHEN** local recovery prepares a new request state with the same `previous_response_id` +- **THEN** the new state retains proxy-injected provenance +- **BUT WHEN** local recovery removes or changes the id +- **THEN** the new state does not inherit that provenance + +#### Scenario: Closed store-false socket quarantines its sent automatic anchor + +- **GIVEN** a sent HTTP bridge request used a proxy-injected `previous_response_id` with `store=false` +- **AND** the upstream socket disconnects before a terminal response +- **WHEN** the disconnect failure is settled +- **THEN** the proxy quarantines that exact anchor before durable release and before mutable replay preparation can erase the id +- **AND** a confirmed clear removes the matching in-memory latest anchor and pending-tool metadata but retains input proof +- **AND** an independently proven safe no-anchor replay may still run +- **AND** the change does not add an ambiguous replay, move accounts, or add an account-health write + +#### Scenario: Disconnect CAS miss preserves in-memory continuity + +- **GIVEN** disconnect invalidation selected an old connection-local anchor +- **AND** the durable owner changed, a newer response advanced, or the persistence write failed +- **WHEN** the conditional quarantine does not confirm a clear +- **THEN** the proxy does not clear the local session's latest-response or pending-tool fields +- **AND** normal fenced-owner or disconnect settlement remains authoritative + +#### Scenario: Disconnect invalidation ignores unsafe candidates + +- **GIVEN** an upstream socket disconnects +- **WHEN** an anchor belongs only to a queued unsent request, is client-supplied, has `store=true`, or conflicts with multiple ambiguous sent anchors +- **THEN** proactive disconnect invalidation does not clear that anchor +- **AND** the normal disconnect settlement remains unchanged + +#### Scenario: Idle disconnect quarantines the current-socket latest response + +- **GIVEN** an HTTP bridge response completed with effective `store=false` +- **AND** its response id remains the session's durable and in-memory latest anchor +- **AND** no request remains pending +- **WHEN** that same upstream socket disconnects +- **THEN** the proxy conditionally quarantines the exact latest response id +- **AND** a confirmed clear removes its in-memory current-socket provenance and pending-tool metadata while retaining input proof + +#### Scenario: Unknown latest-response provenance is not cleared + +- **GIVEN** a session carries a durable latest response id loaded from another socket or process +- **AND** no sent pending request proves that id was used on the current socket +- **WHEN** the current socket disconnects +- **THEN** disconnect handling does not claim the id was produced on that socket +- **AND** the fresh-socket recovery guard remains responsible for preventing automatic reinjection + +#### Scenario: Fresh socket rejects an incremental store-false reattach + +- **GIVEN** a hard durable session retains an automatic latest response id +- **AND** no reusable local socket or forwardable live owner exists +- **WHEN** the client sends incremental or unverifiable input without an explicit anchor +- **THEN** the proxy returns HTTP 400 with code `continuity_requires_full_resend` and parameter `input` +- **AND** the message requests complete context or a new session rather than claiming an upstream close +- **AND** it does not inject the retained connection-local id into the fresh socket +- **AND** it does not create an upstream transport or write account health + +#### Scenario: Owner-forward refresh cannot bypass quarantine + +- **GIVEN** a hard-continuity request was forwarded using an earlier durable owner lookup +- **AND** the owner forward fails before producing output +- **WHEN** the refreshed durable lookup has no latest response id but retains quarantine input proof +- **AND** the request is incremental or otherwise unverifiable without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before local session creation or submission + +#### Scenario: Live recovery socket without a completed anchor rejects incremental input + +- **GIVEN** a hard durable session retains an automatic latest response id +- **AND** a live local recovery socket exists but has not completed that response id on its current socket +- **WHEN** the client sends incremental input without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before upstream submission +- **BUT WHEN** the client sends a fingerprint-verified self-contained full-history resend +- **THEN** the proxy may submit it unanchored on the recovery socket + +#### Scenario: Gate waiter revalidates a connection-local anchor before send + +- **GIVEN** a request serialized a proxy-injected anchor that completed with `store=false` on the current socket +- **AND** the request waits for the response-create gate before it is appended or sent +- **WHEN** that socket is replaced and the request later acquires the gate +- **THEN** the proxy does not send the serialized anchor on the replacement socket +- **AND** it sends a captured unanchored full-history request only when existing replay-safety proof authorizes it +- **AND** an anchor-dependent request receives the full-resend-required HTTP 400 before upstream submission + +#### Scenario: Stale-anchor fallback preserves image preparation + +- **GIVEN** an HTTP bridge request and its replay-safe unanchored fallback contain an external input-image URL +- **AND** bridge image inlining is enabled +- **WHEN** socket replacement invalidates the proxy-injected anchor before send +- **THEN** the selected unanchored fallback contains the inlined image instead of the external URL +- **AND** surviving external URLs fail locally +- **AND** the transformed fallback is rejected locally if it exceeds the upstream serialized request-size budget + +#### Scenario: A different durable id does not inherit socket provenance + +- **GIVEN** a live session records response `resp_local` with current-socket `store=false` provenance +- **WHEN** refreshed durable metadata replaces it with a different id `resp_durable` +- **THEN** the proxy clears the old socket provenance and pending-tool metadata +- **AND** it does not treat `resp_durable` as completed on the current socket diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/specs/responses-api-compat/spec.md b/openspec/changes/recover-codex-desktop-idle-bridge/specs/responses-api-compat/spec.md index 30b2370bcd..8f8c923ca5 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/specs/responses-api-compat/spec.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/specs/responses-api-compat/spec.md @@ -50,3 +50,206 @@ Explicit OpenAI SDK fingerprint markers, including `x-stainless-*` headers or an - **WHEN** the public `/v1/responses` stream normalizer receives an SSE comment keepalive block before a terminal event - **THEN** it forwards the comment keepalive block unchanged - **AND** it continues normalizing the subsequent Responses events normally + +### Requirement: Upstream websocket drops penalize affected accounts + +When an upstream websocket closes while one or more streamed response requests are pending and have not reached a terminal event, the proxy MUST record a transient upstream error for the account before signaling failure for those pending requests, except when the close carries a classified process-wide network failure. A classified process-wide network failure MUST remain account neutral and use its network error code. + +A Responses receive failure without a complete peer close frame MUST be classified as a process-wide network failure when a bounded process-local correlation window observes at least two distinct non-empty upstream account ids fail on the same concrete egress within one second. All candidates in that incident MUST be classified before health settlement as `proxy_network_unavailable`. An owned receive task that has entered bounded correlation MUST complete that decision before a request-budget, stream-idle, or eventless-response deadline can settle the failure. Same-account repeats, different concrete egresses, anonymous accounts, explicit close frames, and live sideband sockets MUST NOT satisfy this correlation rule and MUST retain existing account-health behavior. Correlation MUST NOT make the interrupted post-dispatch request replayable or permit continuity to move across accounts. + +For other closes, the proxy MUST surface `stream_incomplete` to affected pending requests except when a direct Responses WebSocket request has already successfully emitted a finite integer `sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST record the request outcome as `stream_incomplete` without emitting a synthetic terminal frame under the active response id, then MUST close the downstream WebSocket with code 1011. + +#### Scenario: websocket closes before pending responses complete + +- **GIVEN** a streamed response request is pending on an upstream websocket +- **AND** the direct downstream response has not emitted a numeric sequence, or the request uses another transport +- **WHEN** the websocket closes before a terminal response event is observed +- **AND** the close does not carry a classified process-wide network failure +- **THEN** the pending request fails with `stream_incomplete` +- **AND** the account receives a transient upstream failure signal for routing + +#### Scenario: sequenced direct websocket closes before completion + +- **GIVEN** a direct Responses WebSocket request has successfully emitted a finite integer `sequence_number` +- **WHEN** the upstream websocket closes before a terminal response event is observed +- **THEN** the request is recorded as failed with `stream_incomplete` +- **AND** no synthetic terminal frame is emitted under the active response id +- **AND** the downstream WebSocket closes with code 1011 +- **AND** the account receives a transient upstream failure signal for routing + +#### Scenario: correlated no-close failures remain account neutral + +- **GIVEN** pending Responses requests for at least two distinct upstream accounts use the same concrete egress +- **WHEN** their receive paths fail without complete peer close frames within one second +- **THEN** every correlated request fails with `proxy_network_unavailable` +- **AND** no correlated account receives a transient failure signal +- **AND** no request is replayed or moved to another account + +#### Scenario: Deadline settlement waits for observed no-close classification + +- **GIVEN** a direct WebSocket or HTTP bridge receive task has entered bounded no-close correlation +- **WHEN** its request or idle deadline expires before cross-account evidence arrives +- **THEN** the bounded receive classification completes before terminal settlement +- **AND** a correlated network failure is not replaced by a timeout or account-health penalty + +#### Scenario: explicit and uncorrelated closes preserve health behavior + +- **WHEN** a receive failure names only one account, uses a different concrete egress, or carries an explicit close frame +- **THEN** bounded cross-account no-close correlation does not apply +- **AND** the existing close classification and account-health behavior remain authoritative + +## ADDED Requirements + +### Requirement: Eventless durable reattach anchors do not loop forever + +For a hard-continuity HTTP bridge request, a durable `latest_response_id` MAY be injected automatically into a fresh upstream session only while that durable anchor remains trusted. If a request carrying that proxy-injected anchor reaches the eventless missing-`response.created` deadline, the service MUST quarantine the exact durable latest anchor through the fenced compare-and-set behavior defined by proxy admission control. It MUST NOT replay the timed-out request as an anchorless fresh turn. + +Because a `store=false` Responses WebSocket anchor is connection-local, a non-text upstream disconnect or a downstream SSE cancellation that retires the current upstream socket MUST also make an actually sent proxy-injected anchor or an exact latest response proven to have completed on that socket ineligible for later automatic injection. The service MUST apply the protected disconnect-quarantine selection, fenced mutation, and confirmed in-memory matching-anchor clear defined by proxy admission control before durable release and before existing safe no-anchor replay or cancellation retirement may mutate request provenance. It MUST NOT infer current-socket provenance from an unsent request, a client-supplied id, a `store=true` request, a durable id loaded from another socket or process, ambiguous sent anchors, or a failed/fenced persistence mutation. An already-proven safe full-context replay MAY continue without the quarantined anchor, but a canceled request MUST NOT be replayed. + +A hard-continuity request without explicit `previous_response_id` or `conversation` MUST NOT automatically inject a retained `store=false` durable latest-response id onto a fresh WebSocket. A live local session MUST count as a usable continuity path only when the durable latest id matches a response completed with `store=false` on that session's current socket; a live recovery socket without that completion MUST apply the same full-history admission as a fresh socket. The durable row otherwise remains available only for owner routing and full-history recovery proof. A fingerprint-verified self-contained full-context resend MAY start a new unanchored lineage; incremental, prefix-mismatched, or otherwise unverifiable input MUST fail with the full-resend-required client error before an upstream transport is created or a request is submitted. A refreshed lookup after owner-forward failure MUST reapply quarantine admission before local takeover. An explicit client anchor or `conversation` remains distinct from automatic injection. + +An upstream-issued encrypted `compaction` item MAY replace plaintext fingerprint matching only for the same hard-continuity durable fresh-socket or quarantined-anchor recovery. The request MUST omit explicit `previous_response_id` and `conversation`; durable state MUST retain a positive input count, non-empty fingerprint, and concrete owner account; and selection MUST stay fixed to that account. The first input item MUST contain exactly non-blank `id`, literal `type: "compaction"`, and non-blank `encrypted_content`. Remaining input and request controls MUST satisfy the existing account-neutral self-contained fresh-replay validator. The service MUST forward an admitted compaction item unchanged and without the old automatic anchor. It MUST NOT admit malformed items, arbitrary summaries, account-scoped suffix state, missing durable proof or owner, soft-affinity use, ordinary incremental requests, or any cross-account compaction replay. + +If a request waits for response-create admission after a proxy-injected connection-local anchor is serialized, the service MUST revalidate that anchor against current-socket `store=false` completion provenance immediately before enqueue and send. A socket replacement or mismatched durable id MUST NOT carry the serialized anchor across the WebSocket boundary. The service MAY switch to the captured unanchored request only when existing replay-safety proof already marks that full request safe; an anchor-dependent request MUST fail with the full-resend-required client error before upstream submission. When HTTP bridge external-image inlining is enabled, the anchored and captured unanchored candidates MUST both retain that preparation, surviving external-image URLs MUST fail locally, and the serialized size guard MUST apply after transformation to whichever candidate may be sent. + +A later self-contained full-context client resend MUST remain unanchored when durable lookup observes the quarantined state, so it can establish a new upstream response lineage. The quarantined state MUST be derived without a redundant schema field from an absent latest response id together with a retained input count and fingerprint. Before unanchored recovery, the service MUST verify the stored prefix fingerprint and either the existing completed-response safe-full-resend evidence or a quarantine-only self-contained mid-tool continuation. The mid-tool alternative MUST require the projected entire input to have a self-contained call/output graph. The suffix after the projected stored boundary MUST independently satisfy the account-neutral fresh-input validator and MUST contain at least one complete supported direct tool-call/output pair. It MUST NOT require a later assistant-final or new user message. The retained prefix MAY contain existing owner-bound tool declarations only while account selection remains fixed to the durable owner; this alternative MUST NOT make that prefix eligible for account movement. A request without an explicit anchor that is incremental, prefix-mismatched, contains an orphan or incomplete tool call, has unsupported or account-scoped state in its new suffix, or is otherwise not proven self-contained MUST fail closed with the full-resend-required client error before creating or forwarding an upstream request. This alternative MUST NOT relax generic or cross-account replay policy. Historical response-id aliases MAY remain available for explicit client continuity and owner resolution. + +The full-resend-required client error MUST use HTTP 400 with `error.code` equal to `continuity_requires_full_resend`, `error.type` equal to `invalid_request_error`, and `error.param` equal to `input`. Its stable message MUST ask the client to resend complete conversation context in `input` or create a new session and MUST NOT report an upstream WebSocket close. Repeated identical incremental requests against the same quarantined or stale lineage MUST return that same error before transport creation or submission and MUST NOT write account health. Potentially recoverable owner, transport, raw previous-response, and general continuity failures retain their existing retryable error contracts. + +#### Scenario: Full-context resend recovers after eventless durable reattach + +- **GIVEN** a fresh HTTP bridge socket times out before any `response.*` event while using a proxy-injected durable anchor +- **AND** the exact anchor is quarantined successfully +- **WHEN** the client later resends self-contained full context without `previous_response_id` +- **THEN** the proxy verifies the retained input count and fingerprint plus safe-full-resend evidence +- **AND** does not re-inject the quarantined response id +- **AND** the full-context request is forwarded as an unanchored fresh response + +#### Scenario: Completed mid-tool full-history resend recovers after quarantine + +- **GIVEN** durable lookup identifies a quarantined latest-response anchor with retained input count and fingerprint +- **AND** the later unanchored request matches that stored prefix +- **WHEN** the projected suffix contains a complete supported direct tool call and its matching output +- **AND** the projected entire input has a self-contained call/output graph +- **AND** the projected suffix independently satisfies account-neutral fresh-input validation +- **THEN** the proxy forwards the full-context request as an unanchored fresh response +- **AND** recovery does not require a later assistant-final or user message +- **AND** generic and cross-account replay eligibility remain unchanged + +#### Scenario: Malformed mid-tool resend after quarantine fails closed + +- **GIVEN** durable lookup identifies a quarantined latest-response anchor +- **WHEN** a purported full-context resend has a mismatched prefix, orphan tool output, unresolved tool call, duplicate call id, or unsupported/account-scoped state in the new suffix +- **THEN** the proxy returns the full-resend-required HTTP 400 +- **AND** it does not create, forward, or submit an unanchored upstream request + +#### Scenario: Incremental request after quarantine fails closed + +- **GIVEN** durable lookup identifies a quarantined latest-response anchor +- **WHEN** the client sends incremental or prefix-mismatched input without an explicit `previous_response_id` +- **THEN** the proxy returns HTTP 400 with code `continuity_requires_full_resend`, type `invalid_request_error`, and parameter `input` +- **AND** it does not create, forward, or submit an unanchored upstream request + +#### Scenario: Repeated Goal continuation remains a deterministic client error + +- **GIVEN** an automatic durable anchor is quarantined or belongs to a prior socket +- **WHEN** a Goal client repeatedly submits the same incremental continuation without complete context +- **THEN** every attempt returns the same full-resend-required HTTP 400 +- **AND** the message requests complete context or a new session without claiming another upstream close +- **AND** no attempt creates an upstream transport or writes account health + +#### Scenario: Explicit conversation remains independent of automatic-anchor quarantine + +- **GIVEN** durable lookup identifies quarantined automatic response-anchor state +- **WHEN** the client supplies an explicit `conversation` without `previous_response_id` +- **THEN** the proxy does not reject the request solely because the automatic anchor is quarantined +- **AND** the existing explicit-conversation continuity path remains authoritative + +#### Scenario: Owner-forward refresh observes newly quarantined state + +- **GIVEN** an origin forwards a hard-continuity request using an earlier trusted owner lookup +- **WHEN** the forward fails before output and the refreshed lookup has an absent latest response id with retained input proof +- **AND** the request is incremental or unverifiable without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before local session creation or submission + +#### Scenario: Live recovery socket requires socket-local anchor provenance + +- **GIVEN** a live local recovery socket exists for a hard durable session +- **AND** that socket has not completed the durable latest response id with `store=false` +- **WHEN** the client sends incremental input without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before submission +- **BUT WHEN** the client sends a fingerprint-verified self-contained full-history resend +- **THEN** the proxy may submit it unanchored on that socket + +#### Scenario: Socket replacement during gate wait cannot carry an automatic anchor + +- **GIVEN** a request serialized an automatic `store=false` anchor from the current WebSocket +- **AND** it waits for response-create admission +- **WHEN** the WebSocket lineage changes before the request is sent +- **THEN** the proxy does not submit the serialized anchor on the replacement socket +- **AND** a replay-safe full-history request may proceed without the anchor +- **AND** an anchor-dependent request receives the full-resend-required HTTP 400 before submission + +#### Scenario: Image-bearing full-history fallback remains upstream-safe + +- **GIVEN** a replay-safe full-history request has anchored and unanchored forms containing an external input-image URL +- **AND** HTTP bridge image inlining is enabled +- **WHEN** a socket replacement selects the unanchored form at the final send boundary +- **THEN** the selected frame retains the inlined image and contains no surviving external URL +- **AND** the selected transformed frame remains subject to the upstream serialized request-size guard + +#### Scenario: Timed-out incremental request is not replayed fresh + +- **GIVEN** a request depends on a proxy-injected durable anchor and is not independently self-contained +- **WHEN** it reaches the eventless missing-`response.created` deadline +- **THEN** the proxy returns the explicit terminal failure +- **AND** it does not retry that request without `previous_response_id` + +#### Scenario: First full-history retry after a store-false disconnect avoids the dead anchor + +- **GIVEN** an upstream WebSocket disconnects after a sent HTTP bridge request used a proxy-injected `store=false` anchor +- **WHEN** the client retries with verified self-contained full context +- **THEN** the closed socket's exact automatic anchor has already been quarantined +- **AND** the retry follows the existing unanchored quarantine-recovery guard +- **AND** it does not first wait for another missing-`response.created` deadline on the dead connection-local id + +#### Scenario: Escape interruption permits the next verified same-session turn + +- **GIVEN** the client cancels a downstream Codex SSE stream before its upstream response completes +- **AND** retiring the old socket conditionally quarantines its exact eligible automatic `store=false` anchor +- **WHEN** the client sends the next turn under the same session header with fingerprint-matched self-contained full history +- **THEN** the proxy opens a fresh upstream socket and submits that history without the quarantined anchor +- **AND** it does not reconnect or replay the interrupted request + +#### Scenario: Automatic encrypted compaction survives a fresh socket + +- **GIVEN** Codex replaces previously fingerprinted history with an upstream-issued encrypted `compaction` item +- **AND** durable hard-continuity state retains the original owner and prior input proof but no reusable socket +- **WHEN** Codex continues the same session without an explicit anchor +- **THEN** the service opens a fresh socket on that owner and forwards the compaction item unchanged without `previous_response_id` +- **AND** it does not classify the opaque context replacement as an incremental plaintext request + +#### Scenario: Idle close still invalidates the latest connection-local response + +- **GIVEN** the latest response completed with `store=false` on the current WebSocket +- **AND** the socket closes while no request is pending +- **WHEN** the client later sends verified self-contained full history +- **THEN** the completed response id has already been conditionally quarantined +- **AND** the retry starts a new unanchored lineage without a dead-anchor timeout + +#### Scenario: Process restart never reattaches a store-false durable id + +- **GIVEN** a process starts with a hard durable row whose latest response came from a previous WebSocket +- **AND** no live owner can receive the request on that socket +- **WHEN** the client sends a verified self-contained full-history request without an explicit anchor +- **THEN** the proxy forwards the request unanchored on the fresh socket +- **BUT WHEN** the client sends incremental or unverifiable input +- **THEN** the proxy returns the full-resend-required HTTP 400 before opening the upstream transport + +#### Scenario: Soft prompt-cache reconnect does not inherit hard continuity failure + +- **GIVEN** a soft prompt-cache bridge row has quarantined its closed socket's automatic `store=false` anchor +- **AND** the next request supplies no explicit `previous_response_id` +- **WHEN** a self-contained request reaches a fresh upstream socket +- **THEN** the proxy creates the soft-locality session without the quarantined id +- **AND** it does not apply the hard-continuity full-history guard solely because the soft row retains quarantine proof diff --git a/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md b/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md index 1c356ad8d4..90f6efa8b0 100644 --- a/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md +++ b/openspec/changes/recover-codex-desktop-idle-bridge/tasks.md @@ -16,3 +16,87 @@ - [x] 3.1 Run focused bridge and API tests, then Ruff, formatting, type, and architecture checks. - [x] 3.2 Validate the change strictly and validate all repository specs. - [x] 3.3 Review the final diff for secrets/header leakage, account-affinity changes, replay, missing settlement, metric loss, and unrelated edits. + +## 4. Durable anchor quarantine + +- [x] 4.1 Add repository/coordinator regressions for clearing the expected durable latest-response anchor and all coupled metadata while preserving a newer response id or fenced owner. +- [x] 4.2 Add bridge regressions proving the eventless watchdog quarantines only a proxy-injected anchor and a subsequent full-context request does not receive the quarantined id. +- [x] 4.3 Implement the fenced conditional durable-anchor clear and invoke it before missing-`response.created` retirement releases durable ownership. + +## 5. Verification of the quarantine extension + +- [x] 5.1 Run focused durable coordinator, HTTP bridge unit, and HTTP bridge integration tests, then Ruff, formatting, type, and architecture checks. +- [x] 5.2 Validate the change strictly and validate all repository specs. +- [x] 5.3 Review the final diff for stale-write safety, accidental alias deletion, anchorless automatic replay, account-affinity changes, account-health writes, settlement loss, and unrelated edits. + +## 6. Review hardening + +- [x] 6.1 Carry proxy-injected anchor provenance through the owner-forward context and API-to-streaming path. +- [x] 6.2 Bind provenance to the structured HMAC contract, prevent legacy downgrade, and add add/change/strip tamper regressions. +- [x] 6.3 Preserve durable input count and fingerprint during quarantine while clearing the exact response anchor and pending-tool metadata. +- [x] 6.4 Derive quarantine from existing durable fields, allow only a verified safe full-context resend without an anchor, and fail incremental or mismatched requests closed before transport creation. +- [x] 6.5 Add cross-replica propagation, safe full-resend, incremental fail-closed, and fenced persistence regressions. + +## 7. Final verification + +- [x] 7.1 Sync the updated normative requirements and context back to the main capability specs. +- [x] 7.2 Run focused bridge, forwarding, API-contract, and durable coordinator tests, then Ruff, formatting, type, and architecture checks. + +## 8. Mid-tool quarantine recovery hardening + +- [x] 8.1 Specify the fingerprint-matched, self-contained complete tool-call/output recovery path and keep generic/cross-account replay unchanged. +- [x] 8.2 Add pure replay-safety regressions for complete mid-tool suffixes, immediate retries without a new user message, orphan outputs, unresolved calls, duplicate ids, and unsupported state. +- [x] 8.3 Apply the alternative only to the quarantined-anchor guard and add `/backend-api/codex/responses` bridge regressions proving allow and fail-closed outcomes before transport creation. + +## 9. Final verification + +- [ ] 9.1 Validate the OpenSpec change and all repository specs strictly, then complete the Codex review loop. + +## 10. Disconnect invalidation hardening + +- [x] 10.1 Specify the production two-stage disconnect sequence, official connection-local `store=false` recovery contract, and the boundary between proactive anchor invalidation and unsafe transparent replay. +- [x] 10.2 Keep quarantine fail-closed when usable prefix proof is absent, and preserve proxy-injected provenance across same-anchor owner-forward local rebinds. +- [x] 10.3 Record effective request `store` provenance and proactively quarantine only an actually sent, unambiguous proxy-injected `store=false` anchor at upstream disconnect before existing safe no-anchor replay mutates request provenance. +- [x] 10.4 Add repository, owner-forward, selector, and upstream-reader regressions covering sentinel replacement, queued/store/client/newer-owner protection, disconnect close variants, and no new replay/account movement/health writes. + +## 11. Final verification after disconnect hardening + +- [x] 11.1 Sync the normative requirements and operational context to the main specs. +- [ ] 11.2 Run focused and full required checks, strict OpenSpec validation, and the Codex review loop; resolve all Critical/High findings before PR preparation. + +## 12. Idle-close and restart lineage hardening + +- [x] 12.1 Specify current-socket latest-response provenance, idle disconnect quarantine, and the fresh-socket `store=false` lineage boundary. +- [x] 12.2 Record and reset current-socket completion provenance, extend disconnect selection to an idle latest response, and prevent automatic durable-id injection on a fresh socket. +- [x] 12.3 Add idle-close, reconnect/reset, fresh-socket full-history, mid-tool, incremental, and explicit-client-anchor regressions. +- [x] 12.4 Sync the final normative requirements and stable operational context to the main specs. +- [x] 12.5 Revalidate proxy-injected socket provenance after response-create admission, clear provenance when a different durable id replaces local state, and add fail-closed/full-history regressions. + +## 13. Correlated WebSocket egress outage hardening + +- [x] 13.1 Specify bounded same-egress, cross-account no-close correlation and preserve single-account, different-egress, explicit-close, no-replay, and account-affinity behavior. +- [x] 13.2 Add detector, adapter, HTTP bridge, and direct WebSocket regressions proving all correlated candidates remain account neutral while negative controls retain health penalties. +- [x] 13.3 Implement bounded process-local correlation before Responses receive failures reach account-health settlement. +- [x] 13.4 Sync the normative behavior and operational context to the owning main specs. +- [ ] 13.5 Run focused and full required checks, strict OpenSpec validation, final review, and deployment-safety audit. + +## 14. Actionable full-resend-required errors + +- [x] 14.1 Specify and sync the non-retryable HTTP 400 full-resend-required contract for quarantined anchors, fresh-socket automatic anchors, and final send-boundary lineage loss. +- [x] 14.2 Add one dedicated `continuity_requires_full_resend` helper and use it only in the deterministic local guards. +- [x] 14.3 Add unit and backend-route regressions for the stable envelope, repeated Goal-style incremental rejection, no transport creation, and no account-health side effects. +- [ ] 14.4 Run focused and full required checks, strict OpenSpec validation, final review, and deployment-safety audit. + +## 15. Downstream-cancellation lineage invalidation + +- [x] 15.1 Specify Escape/downstream SSE cancellation as a connection-local lineage boundary while preserving the no-replay, no-account-movement ownership barrier. +- [x] 15.2 Snapshot and conditionally quarantine the exact eligible `store=false` automatic anchor before cancellation retirement releases durable ownership, clearing only confirmed matching in-memory provenance. +- [x] 15.3 Add detach and `/backend-api/codex/responses` regressions proving the old socket closes, the interrupted request is not replayed, and the next verified same-session full-history turn succeeds on a fresh socket. +- [x] 15.4 Run focused tests, lint/type/architecture checks, strict OpenSpec validation, and restart/health verification. + +## 16. Automatic encrypted-compaction recovery + +- [x] 16.1 Specify upstream encrypted compaction as a strict same-account complete-context replacement without relaxing plaintext full-resend or cross-account replay policy. +- [x] 16.2 Add a narrow replay-safety classifier and admit it only for hard-continuity durable fresh-socket or quarantine recovery with retained proof and a fixed owner. +- [x] 16.3 Add classifier, bridge-admission, and `/backend-api/codex/responses` regressions covering exact forwarding, owner pinning, malformed items, account-scoped suffixes, ordinary incremental rejection, and no account-health writes. +- [x] 16.4 Run focused and full required checks, strict OpenSpec validation, Codex review, restart/health verification, and PR-readiness checks. diff --git a/openspec/specs/outbound-http-clients/context.md b/openspec/specs/outbound-http-clients/context.md new file mode 100644 index 0000000000..b4ec90d47a --- /dev/null +++ b/openspec/specs/outbound-http-clients/context.md @@ -0,0 +1,53 @@ +# Outbound HTTP Clients Context + +## Purpose and Scope + +This capability owns shared outbound HTTP lifecycle, proxy-aware WebSocket egress, and the boundary between local transport failures and account-specific upstream failures. The normative contract is in `openspec/specs/outbound-http-clients/spec.md`. + +The shared-egress correlation described here is intentionally narrow. It protects account health when one local proxy or direct network path drops several Responses WebSockets at once; it does not change the outcome of the interrupted requests. + +## Decision Rationale + +Typed DNS and route errors already provide strong process-network provenance. A bare WebSocket EOF is different: by itself it may be an account-specific upstream failure, but several nearly simultaneous EOFs from distinct accounts on the same concrete egress are strong evidence that the shared path failed. + +The receive adapter therefore holds an ambiguous no-close Responses failure for up to one second. A second distinct account on the same egress changes every waiting candidate in that incident to the stable `proxy_network_unavailable` classification before account-health settlement. A short retained history gives trailing failures the same classification. One account cannot manufacture the threshold by opening several concurrent requests. + +Concrete egress keys come from structured connection state rather than exception text: + +- routed connections use the actual endpoint id returned after proxy-route fallback; +- environment proxies use parsed scheme, hostname, and port; +- direct connections use the parsed destination scheme, hostname, and port. + +Usernames, passwords, URL paths, access tokens, and raw exception messages are not part of an egress key. Correlation of an ordinary EOF also does not rotate the shared HTTP client because it has not identified a failed shared-client generation. + +## Constraints and Non-Goals + +- Correlation is process-local and bounded; it adds no setting, database state, migration, or cross-replica protocol. +- Only Responses receive failures without a complete peer close frame participate. Live sideband sockets retain their close semantics. +- An already typed process-network error keeps its classification immediately and follows existing transport-rotation behavior. +- Correlation happens after request dispatch. It does not prove whether upstream accepted the request, so it does not authorize replay, account switching, or continuity-owner movement. +- Account ids must be non-empty and distinct. Anonymous failures remain on the established path. + +## Failure Modes and Edge Cases + +- Repeated failures from one broken account time out of the correlation window and retain normal transient health penalties. +- Failures through different routed endpoints, environment proxies, or direct destinations do not corroborate one another. +- A received close frame remains authoritative even when another account closes nearby. +- A downstream keepalive tick does not cancel or duplicate an in-progress receive classification; the relay owns and cleans up one persistent receive task. +- A request-budget, stream-idle, or eventless-response deadline can expire while that persistent task is already classifying an observed EOF. The task stays owned until its bounded decision completes, so the deadline cannot replace a correlated network result with a timeout settlement and account-health write. Truly silent receives that have not observed a failure still obey their normal deadline. +- Cancellation detaches the calling loop's waiter while retaining only bounded incident evidence; expired observations are ignored and removed lazily on subsequent observations. +- Capacity pressure evicts the oldest correlation evidence but does not release that caller before its own bounded judgment window ends. + +## Concrete Incident Example + +At 19:32:10, one environment-proxy EOF ended seven Responses WebSockets across four upstream accounts within 358 milliseconds. One still-valid owner account had three concurrent requests, so three independent `stream_incomplete` health writes crossed its transient-error threshold. Continuity-bound requests then failed with `previous_response_owner_unavailable` until the short local backoff expired, even though the account remained active and had valid credentials and quota. + +With bounded correlation, those seven interrupted requests still fail and are not replayed. Their adapters report `proxy_network_unavailable`, so none of the four accounts receives an error-health or circuit-breaker write and continuity remains pinned to the existing owners. + +For example, if account A observes the shared EOF just before its request budget expires, its relay waits only for the already-started bounded classification. If account B corroborates the same egress during that window, A settles as `proxy_network_unavailable`; the expired budget does not overwrite that transport evidence. + +## Operational Notes + +During rollout, compare clusters of `proxy_network_unavailable` request failures across distinct accounts with account-health counters. A same-egress incident should leave those counters unchanged. A lone `stream_incomplete`, a named close code, or repeated failures from one account should still produce the existing account-specific signal. + +Rollback requires no data conversion because detector observations live only in process memory. Related Responses settlement and continuity behavior is documented in `openspec/specs/responses-api-compat/`. diff --git a/openspec/specs/outbound-http-clients/spec.md b/openspec/specs/outbound-http-clients/spec.md index f682880ac5..a8eb6e723a 100644 --- a/openspec/specs/outbound-http-clients/spec.md +++ b/openspec/specs/outbound-http-clients/spec.md @@ -139,6 +139,8 @@ The service MUST classify local DNS resolver and host-route failures separately The proxy MUST NOT record a transient, permanent, quota, rate-limit, or circuit-breaker health failure against an account when an attempt fails because the local process cannot resolve or route to the upstream host. Routed proxy transport failures MUST retain a credential-safe machine-readable classification after the original exception message is sanitized. A permanent missing proxy hostname MUST remain an endpoint-scoped proxy failure rather than entering process-wide recovery. +A Responses WebSocket receive failure without a complete peer close frame MUST also remain account neutral when a bounded process-local correlation window observes failures from at least two distinct non-empty upstream account ids on the same concrete egress within one second. Concrete egress identity MUST distinguish the actual routed proxy endpoint, parsed environment-proxy endpoint, and direct destination without using exception message text or exposing proxy credentials. Every candidate in the correlated window MUST be classified before account-health settlement as `proxy_network_unavailable`. Downstream keepalive scheduling MUST NOT cancel or restart a pending correlation decision. Once an owned receive task has entered bounded no-close correlation, a request-budget, stream-idle, or eventless-response deadline MUST NOT cancel or settle that receive failure before the correlation decision completes; the completed receive classification MUST reach the existing settlement path first. Repeated failures from one account, failures on different egresses, anonymous accounts, explicit close frames, and live sideband sockets MUST retain their existing classification. Correlation MUST NOT authorize replay of a post-dispatch request, move continuity ownership, or switch accounts. + #### Scenario: Wi-Fi transition does not poison account health - **WHEN** an upstream attempt fails with a classified local DNS or host-route failure @@ -158,6 +160,41 @@ The proxy MUST NOT record a transient, permanent, quota, rate-limit, or circuit- - **THEN** the failure remains `upstream_unavailable` - **AND** the proxy does not classify the host process as disconnected +#### Scenario: Shared egress EOF does not poison account health + +- **GIVEN** two Responses WebSockets for distinct upstream accounts use the same concrete egress +- **WHEN** both receive paths fail without complete peer close frames within one second +- **THEN** every correlated failure carries `proxy_network_unavailable` +- **AND** neither account receives a transient health or circuit-breaker failure +- **AND** no interrupted post-dispatch request is replayed or moved to another account + +#### Scenario: Downstream keepalive does not restart correlation + +- **GIVEN** the configured downstream keepalive interval is shorter than the no-close correlation window +- **WHEN** a Responses receive failure is waiting for its bounded correlation decision +- **THEN** keepalive scheduling does not cancel or restart that receive decision +- **AND** the failure reaches exactly one existing settlement path after correlation completes + +#### Scenario: Request deadline does not preempt in-flight correlation + +- **GIVEN** a Responses receive task has observed a no-close failure and entered bounded correlation +- **WHEN** the owning request budget, stream-idle window, or eventless-response deadline expires before the correlation decision completes +- **THEN** the receive task completes its bounded classification before settlement +- **AND** a correlated `proxy_network_unavailable` result remains account neutral instead of being replaced by a timeout health outcome + +#### Scenario: Single-account and different-egress failures remain account specific + +- **WHEN** no-close receive failures repeat only for one upstream account +- **OR** distinct accounts fail through different concrete egresses +- **THEN** the correlation threshold is not satisfied +- **AND** existing `stream_incomplete` account-health behavior remains authoritative + +#### Scenario: Explicit close frames are not inferred to be a shared EOF + +- **WHEN** an upstream Responses or live sideband WebSocket supplies a close frame +- **THEN** bounded no-close correlation does not reclassify that close +- **AND** the existing close-code and account-health contracts remain authoritative + ### Requirement: Outbound HTTP and WebSocket sessions transparently tunnel through a SOCKS proxy The outbound HTTP and WebSocket clients MUST use a configured SOCKS proxy for all diff --git a/openspec/specs/proxy-admission-control/context.md b/openspec/specs/proxy-admission-control/context.md index a9d11002e1..109fb46b0a 100644 --- a/openspec/specs/proxy-admission-control/context.md +++ b/openspec/specs/proxy-admission-control/context.md @@ -23,8 +23,38 @@ Persistent rebind was rejected because admission completes at different points i Session `S` is mapped to account A. A has all response-create slots in use, while account B has capacity. A new self-contained request carrying only `S` may run on B, but the stored mapping still points to A. A later request that references a response created on B follows that response's hard owner index; it does not rely on `S`. +## Eventless HTTP Bridge Gate Retirement + +The HTTP bridge has both waiter-side and owner-side stuck-gate recovery. Waiter-side recovery handles a later request blocked behind old work; the owner-side watchdog covers the otherwise invisible case where a lone upstream `response.create` send receives no matching response lifecycle event. Its clock begins at the actual send, not at request construction, and is capped at 240 seconds so native clients receive a terminal result before their 300-second parsed-event idle boundary. + +The watchdog is intentionally narrower than the general request and stream timeouts. Leading telemetry does not prove that a response was accepted, while any matched `response.*` event, assigned response id, recorded created latency, or downstream-visible output makes the state ambiguous and leaves the existing timeout paths authoritative. Eligible timeouts fail the whole bridge session closed, settle pending work, and remain account-neutral; automatic replay or account movement could duplicate work whose upstream acceptance is unknown. + +Durable recovery quarantines only the automatic latest-response anchor used by the timed-out send. One owner-and-epoch-fenced compare-and-set clears the anchor and pending-tool metadata while retaining the input count, input fingerprint, and historical aliases. The retained input proof distinguishes a safe self-contained resend from an incremental request that would otherwise lose context. A replacement owner or newer response anchor wins the race, and a persistence error or five-second write timeout is recorded without preventing terminal in-memory settlement. + +Historical rows may lack usable positive input count and fingerprint proof. In that case the same fenced mutation writes a reserved negative count and deterministic non-matching fingerprint instead of nulling all proof. The row therefore remains recognizably quarantined and fails closed until a normally completed response replaces the sentinel with real proof. + +Proxy-injected provenance is part of the signed cross-replica owner-forward context. The structured HMAC binds the marker, and a marked request cannot downgrade to the legacy signature shape. This prevents a receiving owner from mistaking an automatically injected anchor for a client-supplied anchor and skipping quarantine. + +An upstream socket close is an earlier invalidation point for a `store=false` connection-local anchor. While holding the pending-request lock, the bridge first considers actually sent, non-draining HTTP requests with proxy-injected provenance and an unambiguous anchor. It also remembers the effective `store` value of the latest response completed on the current socket, so an idle close can identify the exact latest anchor even after the pending request has been settled. That current-socket provenance is reset whenever the upstream socket changes; a response id loaded from durable state is therefore never guessed to belong to the new socket. The bridge snapshots the selected response id before mutable replay preparation, applies the same fenced compare-and-set, and clears matching in-memory latest-response, socket-provenance, and pending-tool state only after a confirmed durable clear. Queued requests, client-supplied anchors, persisted or unknown-provenance responses, ambiguous candidates, fenced owners, and newer durable progress remain unchanged. + +This early invalidation does not make the interrupted request replayable. The request still follows the existing disconnect settlement because upstream acceptance and tool side effects may be unknown. It only prevents a later verified full-context retry from first reattaching the closed socket's dead automatic anchor and waiting for the eventless watchdog. + +Downstream SSE cancellation (for example, Codex Escape) reaches the same result through a different control path: the proxy intentionally cancels its upstream reader before closing the shared socket, so it must snapshot and quarantine the eligible anchor during detach rather than waiting for a peer-close message the reader can no longer observe. The old socket remains an ownership barrier and the canceled request is never replayed. + +A subsequent Codex automatic compaction can replace that fingerprinted plaintext history with one upstream-issued encrypted `compaction` item. That opaque item is accepted only as same-account complete context: the durable owner stays fixed, the old connection-local response id stays omitted, and any later items must be self-contained and account neutral. A normal text summary or malformed compaction remains subject to the full-resend-required guard. + +A process exit cannot run the idle-close handler, so every truly fresh socket is also a lineage boundary. A retained automatic response id still supplies account-routing and prefix proof, but it is not copied into the new socket. A self-contained full-history resend can start an unanchored lineage after its retained prefix and recovery evidence are verified; an incremental, mismatched, or malformed history fails before transport creation. Reusable local sockets and forwardable live owners keep their current-socket path, while an explicit client-supplied response id remains a separate continuity contract. + +Response-create admission is another socket-lineage race boundary. A request can serialize a valid current-socket anchor and then wait behind the session gate while the reader replaces that socket. The final send path therefore rechecks the id and `store=false` provenance under lifecycle ownership: a previously verified full-history fallback drops the stale anchor, while an incremental continuation fails before it is appended or sent. Both candidates receive the same configured external-image inlining before that choice, including surviving-URL validation and a post-transformation serialized-size check. Otherwise the fallback could restore an `https://` image URL that the anchored frame had already converted, causing the upstream WebSocket to reject or hang. Replacing the local latest-response id from durable metadata also clears provenance belonging to the old id. + +A Goal-enabled production session made the error boundary operationally important. One real disconnect quarantined its anchor, then roughly 57 automatic incremental continuations hit the local full-resend guard. Those attempts never opened an upstream transport, but the former 502 `stream_incomplete` envelope described each one as another WebSocket close, so Goal kept retrying. Deterministic guard failures now use an actionable invalid-request response that asks for complete `input` context or a new session. Owner, network, and other potentially self-healing continuity failures keep their retryable status. + +For example, if session `S` sends at monotonic time 1,000 with durable anchor `resp_old` and produces no response lifecycle event, the default watchdog becomes eligible at 1,240. It returns an explicit failure and conditionally removes `resp_old` from automatic reattach state. The same quarantine can happen immediately if `resp_old` completed with `store=false` and that socket later closes while idle. If the process instead exits before observing the close, the next fresh-socket request still never injects `resp_old`: a self-contained resend can start unanchored only after the retained prefix fingerprint and safe-full-resend checks pass, while an incremental or mismatched resend receives `continuity_requires_full_resend` before upstream transport creation. If that resend contains an external input image and inlining is enabled, both serialized candidates carry the inlined `data:` URL, and the selected unanchored frame is size-checked after conversion. A concurrent advance to `resp_new` is preserved. + ## Operational Notes Operators can distinguish local account pressure through the stable `account_response_create_cap` and `account_stream_cap` reasons. The spillover behavior is zero-config because it mutates no ownership state; rollback restores conservative fail-closed selection without data conversion. -Related capability: `openspec/specs/sticky-session-operations/`. +Monitor the existing stuck-gate retirement counter, the structured `missing_response_created_timeout` detail, upstream `stream_incomplete` close modes, `continuity_requires_full_resend` client-action rejections, and durable-anchor quarantine outcomes for HTTP bridge incidents. A `stream_incomplete` followed by one missing-created timeout on the same conversation/account indicates that invalidation happened one request late; after this change an observed close should quarantine before the client retry, while a process-restart retry should reach the fresh-socket full-history guard directly. Repeated full-resend-required 400s mean the client is still sending incremental context and should not be counted as fresh transport failures or account-health evidence. Compare-and-set misses usually mean ownership or the response lineage advanced safely. No new setting or database migration is involved. + +Related capabilities: `openspec/specs/sticky-session-operations/` and `openspec/specs/responses-api-compat/`. diff --git a/openspec/specs/proxy-admission-control/spec.md b/openspec/specs/proxy-admission-control/spec.md index cc47fa7cac..74eaf61334 100644 --- a/openspec/specs/proxy-admission-control/spec.md +++ b/openspec/specs/proxy-admission-control/spec.md @@ -194,20 +194,257 @@ The dashboard SHALL expose the configured routing policy for each known addition - **THEN** the proxy MAY select that account ### Requirement: Stuck HTTP bridge response-create gate sessions are retired -When a visible HTTP bridge request times out waiting for a per-session response-create gate, the proxy MUST retire the bridge session only if pending visible request age meets or exceeds the configured stuck-gate retirement threshold. The retirement MUST emit a structured low-cardinality log and a Prometheus counter without raw keys or prompt content. + +The proxy MUST retain the existing waiter-triggered retirement behavior for stale HTTP bridge response-create gate owners and MUST additionally enforce an owner-side deadline for a visible HTTP request whose current upstream `response.create` send remains completely eventless before `response.created`. The owner-side deadline MUST be measured from a monotonic timestamp recorded immediately before the current upstream send, MUST use the smaller of the configured stuck-gate retirement threshold and 240 seconds, MUST run without a second gate waiter, and MUST remain active when periodic SSE keepalives are disabled. + +The owner-side watchdog MUST apply only while the request owns the response-create gate, awaits `response.created`, has neither a response id nor recorded `response.created` latency, has received no matched `response.*` lifecycle event, and has produced no downstream-visible output or sequence evidence. Non-response telemetry such as `codex.rate_limits` MUST NOT suppress this watchdog. Any matched `response.*` lifecycle event, response-created milestone, or downstream-visible evidence MUST suppress the owner-side watchdog and leave existing timeout behavior unchanged. + +When the owner-side deadline expires, the proxy MUST recheck eligibility, emit a structured low-cardinality log and the existing stuck-retirement Prometheus counter, terminally fail and settle every pending request exactly once, and retire the whole bridge session. It MUST NOT transparently replay the timed-out request, move it to another account, or write an account-health failure for the missing-created timeout. + +If the expired owner used a proxy-injected `previous_response_id` and the bridge owns a durable session row, the proxy MUST conditionally clear that row's automatic latest-response anchor and pending-tool metadata before releasing durable ownership. It MUST retain the latest input count and input fingerprint as recovery proof. The quarantine MUST be one compare-and-set mutation conditioned on the same session id, owner instance, owner epoch, and expected latest response id. A changed owner, changed epoch, concurrently advanced response id, client-supplied anchor, or nonmatching durable anchor MUST remain unchanged. The quarantine write MUST be bounded to no more than five seconds; a persistence error or timeout MUST be logged and MUST NOT prevent terminal settlement or durable release. + +If the exact anchor has no usable positive input count and non-empty fingerprint, that same compare-and-set mutation MUST write a reserved negative count and deterministic non-empty fingerprint so the row remains detectably quarantined and cannot be mistaken for fresh continuity. Existing usable proof MUST remain unchanged, the reserved proof MUST never satisfy a prefix match, and a later normal completed response MUST replace it with real input proof. + +Proxy-injected anchor provenance MUST survive cross-replica owner forwarding. The forwarding context and reserved header MUST carry the provenance boolean, the canonical structured HMAC payload MUST bind it, and a forward that claims proxy-injected provenance MUST NOT fall back to a legacy signature that does not bind the field. Adding, stripping, or changing the marker MUST either preserve a valid structured signature or reject the forwarded request. + +When an owner-forward attempt fails before yielding and the request is re-prepared for a local session, proxy-injected provenance MUST survive only when the re-prepared `previous_response_id` is exactly equal to the original id. + +When an upstream HTTP bridge WebSocket disconnects, or when downstream SSE cancellation forces the proxy to retire that upstream socket before a peer-close event can be observed, the proxy MUST proactively quarantine an exact connection-local anchor when either an actually sent request used that anchor with effective `store=false` and proxy-injected provenance or the session's latest completed response was produced with effective `store=false` on that same socket. The proxy MUST record current-socket `store` provenance when a response completes and MUST clear that provenance whenever the upstream socket changes. Sent-request candidates MUST be selected from non-draining HTTP requests under the pending lock, MUST prefer a unique current gate owner or an exact session-latest anchor, and MUST act on a single distinct anchor only when no stronger candidate exists. An exact current-socket latest response MAY be selected when no sent candidate exists. The proxy MUST snapshot the response id before marking a canceled request draining and MUST apply the quarantine before existing safe no-anchor replay or cancellation retirement can release durable ownership. When the compare-and-set confirms that exact durable anchor was cleared, the proxy MUST clear the same in-memory latest-response id, its current-socket provenance, and pending-tool metadata while retaining in-memory input count and fingerprint. A CAS miss, persistence failure, fenced owner, or newer durable response MUST NOT clear the in-memory continuity fields. The already-authorized safe replay MAY still run, and a later completed response MUST replace quarantine with its new anchor. Queued or unsent request anchors, `store=true` or unknown-provenance latest responses, client-supplied request anchors, and multiple ambiguous sent anchors MUST remain unchanged. The same owner/epoch/expected-response compare-and-set MUST protect concurrent durable progress. This invalidation MUST NOT replay an interrupted request, reuse its old socket, move accounts, or add an account-health write. + +When a hard-continuity request resolves a durable automatic latest-response id and supplies neither an explicit client `previous_response_id` nor `conversation`, an unanchored incremental request MAY proceed only through a forwardable live owner or a reusable local bridge session whose matching latest response completed with `store=false` on its current socket. The existence of a live local session without that socket-local completion MUST NOT authorize incremental submission. When no such live path exists, the proxy MUST treat the durable id as belonging to a previous socket and MUST NOT inject it. The proxy MAY submit the request unanchored only when the retained count and fingerprint plus the quarantine recovery predicates prove the supplied input is a self-contained full-history resend. Incremental, prefix-mismatched, or otherwise unverifiable input MUST fail closed with the full-resend-required error defined below before upstream transport creation or submission. A refreshed durable lookup after an owner-forward failure MUST reapply the same quarantine and full-resend admission before local takeover. A matching live current-socket response, a forwardable owner, an explicit client-supplied `previous_response_id`, and an explicit `conversation` remain governed by their existing paths. + +As a narrow alternative to plaintext prefix matching, the proxy MAY treat an upstream-issued encrypted `compaction` input item as complete context only for hard-continuity durable fresh-socket or quarantined-anchor recovery. The durable lookup MUST retain a positive input count, a non-empty fingerprint, and a concrete owner account; account selection MUST remain fixed to that owner and MUST NOT fall back or replay across accounts. The request MUST omit explicit `previous_response_id` and `conversation`. Its first input item MUST contain exactly `id`, `type`, and `encrypted_content`, with `type` equal to `compaction` and both other values non-blank strings. Every later input item and all remaining request controls MUST satisfy the existing account-neutral self-contained fresh-replay validator. An admitted request MUST be submitted unanchored with the compaction item preserved exactly. Missing or blank fields, unknown compaction fields, plaintext summaries, account-scoped or non-self-contained suffixes, missing durable proof or owner, soft-affinity requests, and ordinary incremental input MUST NOT use this alternative. + +Immediately after response-create gate acquisition and any closed-session recovery, the proxy MUST revalidate a proxy-injected anchor against the session's current-socket latest response id and `store=false` provenance while lifecycle ownership still excludes socket replacement. If the match was lost during admission, the proxy MUST NOT append or send the serialized anchored request. It MAY instead submit the captured unanchored request only when existing replay-safety proof marks that full request safe; otherwise it MUST fail closed with the full-resend-required error before submission. When HTTP bridge external-image inlining is enabled, both the anchored request and its captured unanchored fallback MUST undergo the same image inlining and surviving-URL validation before this final selection, and each transformed candidate MUST be checked against the upstream serialized request-size budget. When durable lookup replaces a different in-memory latest response id, the proxy MUST clear the previous id's current-socket `store` provenance and pending-tool metadata before the replacement id can influence automatic injection. + +When a quarantined automatic anchor, a retained automatic anchor on a fresh socket, or a final send-boundary lineage mismatch cannot pass the verified full-resend predicate, the proxy MUST return HTTP 400 with `error.code` equal to `continuity_requires_full_resend`, `error.type` equal to `invalid_request_error`, and `error.param` equal to `input`. The stable message MUST instruct the client to resend complete conversation context in `input` or create a new session and MUST NOT claim that an upstream WebSocket just closed. Repeating the same incremental request MUST produce the same local error without creating or submitting an upstream transport and without writing account health. Owner lookup, active-owner availability, transport, raw previous-response recovery, and other general continuity failures MUST retain their existing retryable contracts. #### Scenario: Old pending work blocks a visible gate waiter + - **WHEN** a visible HTTP bridge request receives `response_create_gate_timeout` - **AND** at least one visible pending request on the same session is older than the configured stuck-gate retirement threshold - **THEN** the proxy retires the bridge session so later requests can create a fresh session - **AND** the waiter is rejected cleanly with `response_create_gate_timeout` #### Scenario: Healthy active stream is not retired during a normal wait + - **WHEN** a visible HTTP bridge request times out waiting for the gate - **AND** the session has no pending visible request older than the configured stuck-gate retirement threshold - **THEN** the proxy rejects only the waiter - **AND** the bridge session remains available for the existing in-flight request +#### Scenario: Lone eventless gate owner is retired before the client timeout + +- **GIVEN** a visible HTTP bridge request owns the response-create gate +- **AND** its current `response.create` send produced no matched `response.*` event, response id, or downstream-visible output +- **AND** no second request waits for the gate +- **WHEN** the smaller of the configured stuck threshold and 240 seconds elapses after the current send +- **THEN** the proxy emits an explicit terminal failure and retires the bridge session +- **AND** recovery occurs before the native client's 300-second parsed-event idle timeout + +#### Scenario: Send time rather than request age anchors the deadline + +- **GIVEN** a request spends most of its budget waiting for admission before it sends `response.create` +- **WHEN** the upstream send succeeds +- **THEN** the owner-side deadline begins from that current send +- **AND** earlier queue or admission time does not make the request immediately stale + +#### Scenario: Leading telemetry does not mask an eventless owner + +- **GIVEN** a pre-created gate owner receives `codex.rate_limits` but no matched `response.*` lifecycle event +- **WHEN** the owner-side deadline elapses +- **THEN** the telemetry does not refresh or suppress the deadline +- **AND** the proxy fails and retires the session + +#### Scenario: Response lifecycle evidence suppresses the narrow watchdog + +- **GIVEN** a pre-created request receives any matched `response.*` lifecycle event, a response id, recorded `response.created` latency, or downstream-visible output +- **WHEN** the eventless owner-side deadline would otherwise elapse +- **THEN** this watchdog does not retire the session +- **AND** existing stream, request-budget, and waiter-triggered timeout behavior remains authoritative + +#### Scenario: Timeout is fail-closed and account-neutral + +- **GIVEN** an eventless pre-created owner reaches the owner-side deadline +- **WHEN** terminal cleanup runs +- **THEN** every pending request is settled exactly once and the whole session is retired +- **AND** the proxy does not replay the timed-out request or submit it on another account +- **AND** the selected account is not marked unhealthy solely because `response.created` was missing + +#### Scenario: Eventless proxy-injected anchor is quarantined + +- **GIVEN** an eventless pre-created owner reaches the owner-side deadline +- **AND** its `previous_response_id` was injected by the proxy from the durable session's current latest-response anchor +- **WHEN** terminal retirement runs +- **THEN** the proxy clears that exact durable latest-response anchor and pending-tool metadata using owner-and-epoch fencing +- **AND** it retains the input count and fingerprint as quarantine recovery proof +- **AND** it does not replay the failed request without the anchor + +#### Scenario: Downstream cancellation quarantines before retiring the socket + +- **GIVEN** a downstream HTTP bridge stream is canceled after an eligible proxy-injected `store=false` anchor was sent on the current upstream socket +- **WHEN** detach retires that socket as an upstream ownership barrier +- **THEN** the exact anchor is conditionally quarantined before durable ownership is released +- **AND** confirmed matching in-memory anchor provenance is cleared while input proof is retained +- **AND** the interrupted request is not replayed, the old socket is not reused, and account health is not written + +#### Scenario: Automatic compaction starts a fresh same-account lineage + +- **GIVEN** a hard-continuity durable session retains prior input proof and a concrete owner account +- **AND** no reusable socket or forwardable owner can carry its connection-local automatic anchor +- **WHEN** the client sends an exact encrypted `compaction` item followed only by account-neutral self-contained input +- **THEN** the proxy opens a fresh socket on the durable owner account and omits the old `previous_response_id` +- **AND** it forwards the compaction item unchanged without enabling cross-account replay + +#### Scenario: Cross-replica owner preserves injected-anchor provenance + +- **GIVEN** an origin replica injects a durable `previous_response_id` +- **AND** forwards the request to the active owner replica +- **WHEN** the owner authenticates the internal forwarding context +- **THEN** it marks the owner-side request state as using a proxy-injected anchor +- **AND** an eventless deadline on that owner invokes the same fenced quarantine path + +#### Scenario: Forwarded provenance cannot be downgraded + +- **GIVEN** a signed owner forward carries proxy-injected anchor provenance +- **WHEN** the provenance header is added, stripped, or changed in transit +- **THEN** signature verification rejects the request +- **AND** verification does not fall back to a legacy signature that omits the provenance field + +#### Scenario: Quarantine persistence does not block settlement + +- **GIVEN** an eventless proxy-injected anchor reaches its deadline +- **AND** the durable quarantine write errors or exceeds five seconds +- **WHEN** terminal cleanup runs +- **THEN** the proxy logs the persistence outcome +- **AND** settles every pending request and releases durable ownership without waiting longer + +#### Scenario: Concurrent durable progress survives stale quarantine + +- **GIVEN** an eventless request was sent with proxy-injected anchor `resp_old` +- **AND** durable ownership changed or the durable latest-response anchor advanced after that send +- **WHEN** the stale request reaches the owner-side deadline +- **THEN** the conditional quarantine mutates no durable continuity fields +- **AND** the newer owner or response anchor remains available + +#### Scenario: Explicit client anchor is not quarantined + +- **GIVEN** an eventless pre-created owner carries a client-supplied `previous_response_id` +- **WHEN** the owner-side deadline expires +- **THEN** the proxy retires and settles the ambiguous bridge session +- **AND** it does not clear durable latest-response state solely because the explicit client anchor timed out + +#### Scenario: Missing historical proof remains fail closed after quarantine + +- **GIVEN** the exact proxy-injected anchor has no usable positive input count and fingerprint +- **WHEN** the fenced quarantine clears that anchor +- **THEN** the same mutation stores a reserved non-matching proof pair +- **AND** durable lookup continues to identify the row as quarantined +- **AND** no incoming prefix can match the reserved proof + +#### Scenario: Same-anchor local rebind preserves provenance + +- **GIVEN** an owner-forward request used a proxy-injected anchor +- **AND** forwarding fails before yielding +- **WHEN** local recovery prepares a new request state with the same `previous_response_id` +- **THEN** the new state retains proxy-injected provenance +- **BUT WHEN** local recovery removes or changes the id +- **THEN** the new state does not inherit that provenance + +#### Scenario: Closed store-false socket quarantines its sent automatic anchor + +- **GIVEN** a sent HTTP bridge request used a proxy-injected `previous_response_id` with `store=false` +- **AND** the upstream socket disconnects before a terminal response +- **WHEN** the disconnect failure is settled +- **THEN** the proxy quarantines that exact anchor before durable release and before mutable replay preparation can erase the id +- **AND** a confirmed clear removes the matching in-memory latest anchor and pending-tool metadata but retains input proof +- **AND** an independently proven safe no-anchor replay may still run +- **AND** the change does not add an ambiguous replay, move accounts, or add an account-health write + +#### Scenario: Disconnect CAS miss preserves in-memory continuity + +- **GIVEN** disconnect invalidation selected an old connection-local anchor +- **AND** the durable owner changed, a newer response advanced, or the persistence write failed +- **WHEN** the conditional quarantine does not confirm a clear +- **THEN** the proxy does not clear the local session's latest-response or pending-tool fields +- **AND** normal fenced-owner or disconnect settlement remains authoritative + +#### Scenario: Disconnect invalidation ignores unsafe candidates + +- **GIVEN** an upstream socket disconnects +- **WHEN** an anchor belongs only to a queued unsent request, is client-supplied, has `store=true`, or conflicts with multiple ambiguous sent anchors +- **THEN** proactive disconnect invalidation does not clear that anchor +- **AND** the normal disconnect settlement remains unchanged + +#### Scenario: Idle disconnect quarantines the current-socket latest response + +- **GIVEN** an HTTP bridge response completed with effective `store=false` +- **AND** its response id remains the session's durable and in-memory latest anchor +- **AND** no request remains pending +- **WHEN** that same upstream socket disconnects +- **THEN** the proxy conditionally quarantines the exact latest response id +- **AND** a confirmed clear removes its in-memory current-socket provenance and pending-tool metadata while retaining input proof + +#### Scenario: Unknown latest-response provenance is not cleared + +- **GIVEN** a session carries a durable latest response id loaded from another socket or process +- **AND** no sent pending request proves that id was used on the current socket +- **WHEN** the current socket disconnects +- **THEN** disconnect handling does not claim the id was produced on that socket +- **AND** the fresh-socket recovery guard remains responsible for preventing automatic reinjection + +#### Scenario: Fresh socket rejects an incremental store-false reattach + +- **GIVEN** a hard durable session retains an automatic latest response id +- **AND** no reusable local socket or forwardable live owner exists +- **WHEN** the client sends incremental or unverifiable input without an explicit anchor +- **THEN** the proxy returns HTTP 400 with code `continuity_requires_full_resend` and parameter `input` +- **AND** the message requests complete context or a new session rather than claiming an upstream close +- **AND** it does not inject the retained connection-local id into the fresh socket +- **AND** it does not create an upstream transport or write account health + +#### Scenario: Owner-forward refresh cannot bypass quarantine + +- **GIVEN** a hard-continuity request was forwarded using an earlier durable owner lookup +- **AND** the owner forward fails before producing output +- **WHEN** the refreshed durable lookup has no latest response id but retains quarantine input proof +- **AND** the request is incremental or otherwise unverifiable without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before local session creation or submission + +#### Scenario: Live recovery socket without a completed anchor rejects incremental input + +- **GIVEN** a hard durable session retains an automatic latest response id +- **AND** a live local recovery socket exists but has not completed that response id on its current socket +- **WHEN** the client sends incremental input without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before upstream submission +- **BUT WHEN** the client sends a fingerprint-verified self-contained full-history resend +- **THEN** the proxy may submit it unanchored on the recovery socket + +#### Scenario: Gate waiter revalidates a connection-local anchor before send + +- **GIVEN** a request serialized a proxy-injected anchor that completed with `store=false` on the current socket +- **AND** the request waits for the response-create gate before it is appended or sent +- **WHEN** that socket is replaced and the request later acquires the gate +- **THEN** the proxy does not send the serialized anchor on the replacement socket +- **AND** it sends a captured unanchored full-history request only when existing replay-safety proof authorizes it +- **AND** an anchor-dependent request receives the full-resend-required HTTP 400 before upstream submission + +#### Scenario: Stale-anchor fallback preserves image preparation + +- **GIVEN** an HTTP bridge request and its replay-safe unanchored fallback contain an external input-image URL +- **AND** bridge image inlining is enabled +- **WHEN** socket replacement invalidates the proxy-injected anchor before send +- **THEN** the selected unanchored fallback contains the inlined image instead of the external URL +- **AND** surviving external URLs fail locally +- **AND** the transformed fallback is rejected locally if it exceeds the upstream serialized request-size budget + +#### Scenario: A different durable id does not inherit socket provenance + +- **GIVEN** a live session records response `resp_local` with current-socket `store=false` provenance +- **WHEN** refreshed durable metadata replaces it with a different id `resp_durable` +- **THEN** the proxy clears the old socket provenance and pending-tool metadata +- **AND** it does not treat `resp_durable` as completed on the current socket + ### Requirement: Account stream capacity reserves recovery headroom The proxy MUST reserve the configured number of account-local stream slots from ordinary first-turn and follow-up selection, while allowing reattach work to use the full account stream cap. The default recovery reserve MUST be one slot. The reserve MUST NOT increase the configured hard stream cap. diff --git a/openspec/specs/responses-api-compat/context.md b/openspec/specs/responses-api-compat/context.md index 3eb8c29881..afa7fd3f07 100644 --- a/openspec/specs/responses-api-compat/context.md +++ b/openspec/specs/responses-api-compat/context.md @@ -33,6 +33,50 @@ See `openspec/specs/responses-api-compat/spec.md` for normative requirements. - `prompt_cache_key` affinity on OpenAI-style routes is intentionally bounded by a dashboard-managed freshness window, unlike durable backend `session_id` or dashboard sticky-thread routing. - Codex-native direct websocket `/backend-api/codex/responses` treats upstream `previous_response_id` as an ephemeral anchor. If that anchor goes stale, the proxy must mask raw `previous_response_not_found` details and emit a sanitized `codex_previous_response_stale` classifier so compatible Codex clients can soft-reset and retry without `previous_response_id`. +## HTTP Bridge Liveness and Durable Anchor Recovery + +HTTP bridge heartbeat framing uses client identity separately from event-shape normalization. A verified native Codex Desktop request needs a parsed JSON SSE event before `response.created`, even when its payload also looks OpenAI-compatible and therefore still passes through response-event normalization. Explicit OpenAI SDK fingerprints take precedence and keep comment heartbeats, while public `/v1/responses` continues exposing only OpenAI-contract-safe events. + +This split avoids treating payload heuristics or continuity headers as client authentication. It changes only the liveness frame: authentication, validation, routing, fingerprint normalization, and public vendor-event filtering continue through their existing paths. The first generated heartbeat still waits until after the startup-error probe so a local startup failure can retain its HTTP status. + +Durable `latest_response_id` is an optimization for automatically reattaching a fresh upstream socket, not an indefinitely trusted conversation record. If an automatically injected anchor produces no response lifecycle event before the owner watchdog expires, the exact anchor is quarantined from later automatic injection. Quarantine retains the prior input count and fingerprint as recovery proof, while historical aliases remain available for explicit continuity lookups and fencing preserves a newer owner or response lineage. + +OpenAI's [WebSocket conversation-state guidance](https://developers.openai.com/api/docs/guides/conversation-state#previous_response_id-in-websocket-mode) and [deployment checklist](https://developers.openai.com/api/docs/guides/deployment-checklist#use-websocket-mode) define the upstream boundary: a WebSocket currently has a maximum lifetime of 60 minutes, its most recent response cache is connection-local, and an uncached `store=false` chain must reconnect with `previous_response_id` omitted plus full input context. A response id learned on a closed `store=false` socket is therefore not durable reattach state even if codex-lb persisted it for bridge reuse. + +One overnight production aggregate contained 10 client-visible 502 events across seven Codex conversations and three upstream accounts. Every affected conversation had exactly one approximately 240-second `missing_response_created_timeout`. In the later five conversations that timeout immediately followed `stream_incomplete` on the same conversation and account; observed closes included no close frame, code 1000, and code 1001. Active conversations later succeeded and none repeated the missing-created timeout after quarantine. Two early cases followed an intentional local restart, but the later sequence occurred without another process restart. This establishes a two-stage failure: the upstream socket rotates or closes, then codex-lb re-injects that socket's connection-local anchor into the fresh socket. + +Disconnect-time quarantine removes the second stage. It first considers an actually sent, unambiguous, proxy-injected `store=false` anchor. The bridge also records whether its latest completed response was created with `store=false` on the current socket, which lets an idle close quarantine that exact response after the request queue is empty. This completion provenance is cleared whenever the upstream socket changes, so a durable id loaded from another socket is not mistaken for local evidence. Both paths use the same owner/epoch/expected-response compare-and-set. A confirmed clear also removes the matching in-memory latest anchor, its socket provenance, and pending-tool metadata; a CAS miss or persistence error leaves local continuity untouched. The request interrupted by the close still fails normally because transparent replay could duplicate model output or custom-tool side effects. The safety benefit is that the client's next verified full-history retry reaches the existing unanchored quarantine guard immediately instead of spending another 240 seconds on the dead anchor. + +A crash or process restart cannot run the disconnect handler. The fresh-socket admission rule therefore treats any retained automatic `store=false` response id as routing and recovery proof, not as an id that can be copied into a new WebSocket. The same prefix-fingerprint, completed-output, and self-contained mid-tool predicates used by quarantine recovery decide whether a client-supplied full history may start unanchored. Incremental or unverifiable input fails before the new upstream transport is opened. A reusable local socket, a forwardable live owner, and an explicit client anchor keep their existing continuity paths. + +Codex Escape closes the downstream SSE before the upstream peer necessarily emits a close frame. Because bridge detach cancels the reader while retiring that socket, it performs the same fenced anchor quarantine directly before durable release. The next fingerprint-verified same-session full-history turn can therefore establish a fresh unanchored lineage; the interrupted turn itself is not replayed. + +Codex may instead compact a long session into an upstream-issued encrypted `compaction` item. The ciphertext intentionally does not match the pre-compaction plaintext fingerprint, so fresh-socket admission recognizes its exact protocol shape as complete context while keeping the durable owner account fixed. For example, `[{"id":"cmp_1","type":"compaction","encrypted_content":"..."}]` can start the replacement lineage without the old response id; a user-authored summary, an empty ciphertext, or a suffix containing an account-scoped file id still fails closed. + +The same lineage check is repeated after response-create gate admission. This closes the window where an anchor was valid during request preparation but its socket disconnected while the request waited. The bridge sends the captured unanchored form only when existing replay-safety proof permits it; otherwise it returns the actionable full-resend-required 400 without placing the anchored frame on the replacement socket. When external-image inlining is enabled, request preparation applies it to both the anchored and captured unanchored forms and enforces the serialized-size guard after each transformation. This prevents the late fallback choice from reintroducing an external URL that the upstream WebSocket cannot accept. + +Soft prompt-cache affinity remains locality rather than hard conversation ownership. If an idle close quarantines its connection-local response id, the next self-contained prompt-cache request may create a fresh session without that id; retained quarantine proof alone does not turn the soft key into a hard full-history requirement. + +For example, after a restart the durable row for a Desktop session may still contain `resp_old` plus a fingerprint of its prior input. A rebuilt self-contained history matching that fingerprint is sent on the fresh socket without `resp_old` and can establish the next lineage. A one-item incremental continuation is rejected with HTTP 400 `continuity_requires_full_resend` before connection creation. The message asks the client to resend complete context in `input` or create a new session instead of reporting another upstream close. If `resp_old` instead completed on a still-running socket and that socket closes while idle, current-socket completion provenance lets the close handler quarantine it immediately. + +Codex CLI can retry while a turn is still progressing through tools. A normal rebuilt history may append assistant commentary, a `custom_tool_call`, and its matching `custom_tool_call_output` without yet appending an assistant-final or another user message. For quarantine recovery only, that is valid fresh-context evidence when the durable prefix fingerprint matches, the projected whole history has a self-contained call/output graph, and the projected suffix independently passes strict account-neutral fresh-input validation. The retained prefix may keep its existing `additional_tools` declarations because recovery stays on the durable owner account; it is not made portable across accounts. An orphan output, unresolved call, duplicate id, unsupported/account-scoped suffix item, or ordinary incremental message remains fail-closed. This exception does not broaden cross-account replay. + +When an origin replica injected the anchor but another replica owns the bridge, the forwarding HMAC binds that provenance marker. Adding, stripping, or changing it invalidates the structured signature, and marked forwards cannot fall back to an older signature that omitted the security-relevant field. + +One Goal-enabled production session exposed why this distinction must be machine-readable. After a single real transport failure quarantined the old lineage, roughly 57 automatic incremental continuations were rejected locally. The former `stream_incomplete` 502 made every rejection look like another transient WebSocket close, so Goal retried without changing the input. The dedicated invalid-request code makes the required client action explicit and leaves genuinely transient owner, network, and general continuity failures on their prior retryable paths. + +Operationally, correlate `missing_response_created_timeout` retirement logs with stuck-gate metrics, idle/sent disconnect quarantine outcomes, and fresh-socket continuity rejections. Repeated timeouts with successful quarantine point to upstream acceptance or socket-liveness problems; compare-and-set misses usually mean another owner or response already advanced safely. A fresh-socket `continuity_requires_full_resend` after restart means the client did not supply a verifiable full history, not that the retained response id or identical request should be retried. No migration or new runtime setting is required. + +## Shared-Egress WebSocket Disconnect Classification + +A later production incident showed that ordinary per-account disconnect handling can amplify one local network fault. Seven Responses WebSockets across four accounts ended within 358 milliseconds after one shared environment-proxy EOF. Three concurrent requests belonged to the same healthy continuity owner, so their independent `stream_incomplete` writes crossed the transient-error threshold and temporarily turned later owner-pinned retries into `previous_response_owner_unavailable`. + +The adapter now delays only ambiguous Responses receive failures that have no complete peer close frame. Cross-account evidence on the same credential-safe egress key changes the incident to `proxy_network_unavailable` before the HTTP bridge or direct WebSocket relay reaches health settlement. Once a receive task has entered that bounded decision, a concurrent request, idle, or eventless deadline waits for the observed transport failure to finish classification; a truly silent receive remains governed by its normal deadline. Routed fallback uses the actual endpoint id; environment proxies and direct destinations use parsed endpoint components without credentials. The detailed egress decision and edge cases live in `openspec/specs/outbound-http-clients/context.md`. + +This classification does not make the interrupted request safe to replay. The send already completed, so upstream acceptance and tool or model side effects remain unknown. Both bridge and direct-WebSocket paths return the network failure without moving accounts or continuity ownership. A single-account failure, a different egress, an anonymous account, a received close frame, or a Live sideband socket retains the previous `stream_incomplete` and health behavior. + +Operationally, a correlated burst should appear as near-simultaneous `proxy_network_unavailable` outcomes for distinct accounts without matching account-health counter increases, even when one affected request reaches its local deadline during the one-second decision window. A named close code or repeated `stream_incomplete` on one account remains evidence for the normal account-specific path. The detector is bounded and process-local, requires no setting or migration, and disappears on restart. + ## Fast Mode and Service Tiers codex-lb accepts the OpenAI/Codex `service_tier` field on Responses and Chat @@ -110,6 +154,7 @@ when upstream reports a different actual tier. - **Upstream error / no accounts:** Non-streaming responses return an OpenAI error envelope with 5xx status. - **Compact upstream transport/client failure:** Retry only inside `/codex/responses/compact` when the failure is safely retryable; otherwise return an explicit upstream error without surrogate fallback. - **HTTP bridge session closes or expires:** The next compatible HTTP `/v1/responses` or `/backend-api/codex/responses` request recreates a fresh upstream websocket bridge session; continuity is guaranteed only within the lifetime of one active bridged session. +- **Automatic anchor requires complete context:** Deterministic quarantine and fresh-socket lineage guards return HTTP 400 `continuity_requires_full_resend` with `param=input`; resend complete context or create a new session instead of retrying the same incremental input. - **Multi-instance routing without bridge owner policy:** if operators do not configure a bridge ring or front-door affinity, continuity can still fragment across replicas. With a configured bridge ring, hard continuity keys landing on a non-owner replica are proxy-forwarded to the owner replica; the proxy fails closed only when the owner endpoint or ring membership cannot be resolved or the forward signature fails authentication. Gateway-safe prompt-cache requests may accept locality misses and continue locally instead of forwarding. - **Codex websocket reconnects:** Reconnect continuity now depends on the client replaying the accepted `x-codex-turn-state`; generated turn-state is emitted on accept for backend Codex routes and echoed back when the client already supplies one. - **Codex websocket stale previous-response anchors:** Direct backend Codex websocket stale-anchor failures are surfaced as `response.failed` / `codex_previous_response_stale` without the raw upstream code or missing `resp_...` id; OpenAI-compatible `/v1/responses` websocket clients continue to receive generic `stream_incomplete` masking. @@ -118,6 +163,7 @@ when upstream reports a different actual tier. ## Error Envelope Mapping (Reference) +- 400 full-resend guard → `continuity_requires_full_resend` / `invalid_request_error` - 401 → `invalid_api_key` - 403 → `insufficient_permissions` - 404 → `not_found` diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index 3e871142ab..ae865b1de1 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -84,7 +84,11 @@ The default compact request budget MUST be at least 180 seconds, and the default - **AND** `stream_idle_timeout_seconds` is at least 600 seconds ### Requirement: Upstream websocket drops penalize affected accounts -When an upstream websocket closes while one or more streamed response requests are pending and have not reached a terminal event, the proxy MUST record a transient upstream error for the account before signaling failure for those pending requests, except when the close carries a classified process-wide network failure. A classified process-wide network failure MUST remain account neutral and use its network error code. For other closes, the proxy MUST surface `stream_incomplete` to affected pending requests except when a direct Responses WebSocket request has already successfully emitted a finite integer `sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST record the request outcome as `stream_incomplete` without emitting a synthetic terminal frame under the active response id, then MUST close the downstream WebSocket with code 1011. +When an upstream websocket closes while one or more streamed response requests are pending and have not reached a terminal event, the proxy MUST record a transient upstream error for the account before signaling failure for those pending requests, except when the close carries a classified process-wide network failure. A classified process-wide network failure MUST remain account neutral and use its network error code. + +A Responses receive failure without a complete peer close frame MUST be classified as a process-wide network failure when a bounded process-local correlation window observes at least two distinct non-empty upstream account ids fail on the same concrete egress within one second. All candidates in that incident MUST be classified before health settlement as `proxy_network_unavailable`. An owned receive task that has entered bounded correlation MUST complete that decision before a request-budget, stream-idle, or eventless-response deadline can settle the failure. Same-account repeats, different concrete egresses, anonymous accounts, explicit close frames, and live sideband sockets MUST NOT satisfy this correlation rule and MUST retain existing account-health behavior. Correlation MUST NOT make the interrupted post-dispatch request replayable or permit continuity to move across accounts. + +For other closes, the proxy MUST surface `stream_incomplete` to affected pending requests except when a direct Responses WebSocket request has already successfully emitted a finite integer `sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST record the request outcome as `stream_incomplete` without emitting a synthetic terminal frame under the active response id, then MUST close the downstream WebSocket with code 1011. #### Scenario: websocket closes before pending responses complete @@ -104,6 +108,27 @@ When an upstream websocket closes while one or more streamed response requests a - **AND** the downstream WebSocket closes with code 1011 - **AND** the account receives a transient upstream failure signal for routing +#### Scenario: correlated no-close failures remain account neutral + +- **GIVEN** pending Responses requests for at least two distinct upstream accounts use the same concrete egress +- **WHEN** their receive paths fail without complete peer close frames within one second +- **THEN** every correlated request fails with `proxy_network_unavailable` +- **AND** no correlated account receives a transient failure signal +- **AND** no request is replayed or moved to another account + +#### Scenario: Deadline settlement waits for observed no-close classification + +- **GIVEN** a direct WebSocket or HTTP bridge receive task has entered bounded no-close correlation +- **WHEN** its request or idle deadline expires before cross-account evidence arrives +- **THEN** the bounded receive classification completes before terminal settlement +- **AND** a correlated network failure is not replaced by a timeout or account-health penalty + +#### Scenario: explicit and uncorrelated closes preserve health behavior + +- **WHEN** a receive failure names only one account, uses a different concrete egress, or carries an explicit close frame +- **THEN** bounded cross-account no-close correlation does not apply +- **AND** the existing close classification and account-health behavior remain authoritative + ### Requirement: Single HTTP bridge previous-response misses recover or fail closed When an HTTP bridge session receives an anonymous upstream `previous_response_not_found` error for a single pending follow-up request, the service MUST treat the error as an internal continuity-loss signal. It MUST either recover through the existing previous-response rebind path or rewrite the error to a retryable continuity failure instead of forwarding the raw upstream invalid-request error. @@ -836,7 +861,12 @@ When an upstream websocket or HTTP bridge session has multiple pending Responses - **AND** the younger request remains pending ### Requirement: HTTP bridge streams emit downstream liveness frames while pending -When an HTTP bridge Responses request is waiting for upstream queue events, the system MUST emit a downstream SSE liveness frame at the configured `sse_keepalive_interval_seconds` interval so downstream clients do not disconnect before the upstream terminal frame arrives. The first generated liveness frame MUST be delayed until after the HTTP bridge startup-error probe window so a local startup `ProxyResponseError` can still be surfaced as a non-2xx HTTP response. Once a generated liveness frame is emitted, the stream MUST be considered started for later HTTP-error propagation decisions, so a subsequent upstream `response.failed` is forwarded in-stream instead of being raised as a startup HTTP error. If the pending request already has a response id, the liveness frame MAY be a `response.in_progress` SSE event for that response id. If no response id is known yet, the Codex CLI route MUST emit an ignored `codex.keepalive` SSE data event because comment-only frames do not reset the CLI's EventSource idle timer. Public `/v1/responses` stream normalization MUST preserve SSE comment keepalives instead of treating them as malformed data, and MUST drop `codex.*` liveness events from the public OpenAI SDK contract surface. + +When an HTTP bridge Responses request is waiting for upstream queue events, the system MUST emit a downstream SSE liveness frame at the configured `sse_keepalive_interval_seconds` interval so downstream clients do not disconnect before the upstream terminal frame arrives. The first generated liveness frame MUST be delayed until after the HTTP bridge startup-error probe window so a local startup `ProxyResponseError` can still be surfaced as a non-2xx HTTP response. Once a generated liveness frame is emitted, the stream MUST be considered started for later HTTP-error propagation decisions, so a subsequent upstream `response.failed` is forwarded in-stream instead of being raised as a startup HTTP error. + +If the pending request already has a response id, the liveness frame MAY be a `response.in_progress` SSE event for that response id. Before a response id exists, a verified native Codex client on `/backend-api/codex/responses` MUST receive an event-bearing `codex.keepalive` JSON SSE frame even when payload-shape heuristics also require OpenAI-compatible response normalization, because comment-only frames do not reset the native client's parsed-event idle timer. Native identity MUST come from the existing native User-Agent or originator allowlist and MUST NOT be inferred from continuity headers. + +Explicit OpenAI SDK fingerprint markers, including `x-stainless-*` headers or an OpenAI User-Agent, MUST retain precedence for heartbeat framing and MUST receive comment liveness. Public `/v1/responses` and other non-native OpenAI SDK streams MUST retain comment heartbeats before `response.created`; public stream normalization MUST preserve those comments and MUST drop `codex.*` liveness events from the OpenAI contract surface. Heartbeat selection MUST NOT disable authentication, payload validation, event normalization, fingerprint normalization, or routing policy. #### Scenario: HTTP bridge emits response in-progress keepalive after response id is known - **GIVEN** an HTTP bridge request has a known response id @@ -844,11 +874,27 @@ When an HTTP bridge Responses request is waiting for upstream queue events, the - **THEN** the downstream stream emits a `response.in_progress` event for that response id - **AND** the request remains pending -#### Scenario: HTTP bridge emits Codex keepalive before response id is known -- **GIVEN** an HTTP bridge request does not yet have a response id -- **WHEN** no upstream event arrives before the SSE keepalive interval elapses -- **THEN** the downstream stream emits a `codex.keepalive` SSE data event -- **AND** the request remains pending +#### Scenario: Native Desktop shape receives parsed-event liveness + +- **GIVEN** Codex Desktop sends `POST /backend-api/codex/responses` with a verified native User-Agent or originator +- **AND** its OpenAI-compatible payload and `Accept` header also trigger SDK-compatible event normalization +- **WHEN** no upstream event arrives before a response id is known +- **THEN** the proxy emits an event-bearing `codex.keepalive` JSON SSE frame +- **AND** it preserves any required response-event normalization + +#### Scenario: Explicit SDK marker retains comment liveness + +- **GIVEN** a request to `/backend-api/codex/responses` carries an `x-stainless-*` header or OpenAI User-Agent +- **WHEN** its payload also resembles a native Codex request +- **THEN** the proxy emits an SSE comment heartbeat before `response.created` +- **AND** it does not expose `codex.*` vendor events to the SDK stream + +#### Scenario: Public v1 route never exposes native vendor heartbeat + +- **GIVEN** a request targets public `/v1/responses` +- **WHEN** the request is pending before `response.created` +- **THEN** periodic liveness uses OpenAI-contract-safe comment frames +- **AND** the first data event remains `response.created` #### Scenario: First HTTP bridge keepalive is delayed past startup probe - **GIVEN** an HTTP bridge request is waiting for upstream queue events @@ -868,6 +914,160 @@ When an HTTP bridge Responses request is waiting for upstream queue events, the - **THEN** it forwards the comment keepalive block unchanged - **AND** it continues normalizing the subsequent Responses events normally +### Requirement: Eventless durable reattach anchors do not loop forever + +For a hard-continuity HTTP bridge request, a durable `latest_response_id` MAY be injected automatically into a fresh upstream session only while that durable anchor remains trusted. If a request carrying that proxy-injected anchor reaches the eventless missing-`response.created` deadline, the service MUST quarantine the exact durable latest anchor through the fenced compare-and-set behavior defined by proxy admission control. It MUST NOT replay the timed-out request as an anchorless fresh turn. + +Because a `store=false` Responses WebSocket anchor is connection-local, a non-text upstream disconnect or a downstream SSE cancellation that retires the current upstream socket MUST also make an actually sent proxy-injected anchor or an exact latest response proven to have completed on that socket ineligible for later automatic injection. The service MUST apply the protected disconnect-quarantine selection, fenced mutation, and confirmed in-memory matching-anchor clear defined by proxy admission control before durable release and before existing safe no-anchor replay or cancellation retirement may mutate request provenance. It MUST NOT infer current-socket provenance from an unsent request, a client-supplied id, a `store=true` request, a durable id loaded from another socket or process, ambiguous sent anchors, or a failed/fenced persistence mutation. An already-proven safe full-context replay MAY continue without the quarantined anchor, but a canceled request MUST NOT be replayed. + +A hard-continuity request without explicit `previous_response_id` or `conversation` MUST NOT automatically inject a retained `store=false` durable latest-response id onto a fresh WebSocket. A live local session MUST count as a usable continuity path only when the durable latest id matches a response completed with `store=false` on that session's current socket; a live recovery socket without that completion MUST apply the same full-history admission as a fresh socket. The durable row otherwise remains available only for owner routing and full-history recovery proof. A fingerprint-verified self-contained full-context resend MAY start a new unanchored lineage; incremental, prefix-mismatched, or otherwise unverifiable input MUST fail with the full-resend-required client error before an upstream transport is created or a request is submitted. A refreshed lookup after owner-forward failure MUST reapply quarantine admission before local takeover. An explicit client anchor or `conversation` remains distinct from automatic injection. + +An upstream-issued encrypted `compaction` item MAY replace plaintext fingerprint matching only for the same hard-continuity durable fresh-socket or quarantined-anchor recovery. The request MUST omit explicit `previous_response_id` and `conversation`; durable state MUST retain a positive input count, non-empty fingerprint, and concrete owner account; and selection MUST stay fixed to that account. The first input item MUST contain exactly non-blank `id`, literal `type: "compaction"`, and non-blank `encrypted_content`. Remaining input and request controls MUST satisfy the existing account-neutral self-contained fresh-replay validator. The service MUST forward an admitted compaction item unchanged and without the old automatic anchor. It MUST NOT admit malformed items, arbitrary summaries, account-scoped suffix state, missing durable proof or owner, soft-affinity use, ordinary incremental requests, or any cross-account compaction replay. + +If a request waits for response-create admission after a proxy-injected connection-local anchor is serialized, the service MUST revalidate that anchor against current-socket `store=false` completion provenance immediately before enqueue and send. A socket replacement or mismatched durable id MUST NOT carry the serialized anchor across the WebSocket boundary. The service MAY switch to the captured unanchored request only when existing replay-safety proof already marks that full request safe; an anchor-dependent request MUST fail with the full-resend-required client error before upstream submission. When HTTP bridge external-image inlining is enabled, the anchored and captured unanchored candidates MUST both retain that preparation, surviving external-image URLs MUST fail locally, and the serialized size guard MUST apply after transformation to whichever candidate may be sent. + +A later self-contained full-context client resend MUST remain unanchored when durable lookup observes the quarantined state, so it can establish a new upstream response lineage. The quarantined state MUST be derived without a redundant schema field from an absent latest response id together with a retained input count and fingerprint. Before unanchored recovery, the service MUST verify the stored prefix fingerprint and either the existing completed-response safe-full-resend evidence or a quarantine-only self-contained mid-tool continuation. The mid-tool alternative MUST require the projected entire input to have a self-contained call/output graph. The suffix after the projected stored boundary MUST independently satisfy the account-neutral fresh-input validator and MUST contain at least one complete supported direct tool-call/output pair. It MUST NOT require a later assistant-final or new user message. The retained prefix MAY contain existing owner-bound tool declarations only while account selection remains fixed to the durable owner; this alternative MUST NOT make that prefix eligible for account movement. A request without an explicit anchor that is incremental, prefix-mismatched, contains an orphan or incomplete tool call, has unsupported or account-scoped state in its new suffix, or is otherwise not proven self-contained MUST fail closed with the full-resend-required client error before creating or forwarding an upstream request. This alternative MUST NOT relax generic or cross-account replay policy. Historical response-id aliases MAY remain available for explicit client continuity and owner resolution. + +The full-resend-required client error MUST use HTTP 400 with `error.code` equal to `continuity_requires_full_resend`, `error.type` equal to `invalid_request_error`, and `error.param` equal to `input`. Its stable message MUST ask the client to resend complete conversation context in `input` or create a new session and MUST NOT report an upstream WebSocket close. Repeated identical incremental requests against the same quarantined or stale lineage MUST return that same error before transport creation or submission and MUST NOT write account health. Potentially recoverable owner, transport, raw previous-response, and general continuity failures retain their existing retryable error contracts. + +#### Scenario: Full-context resend recovers after eventless durable reattach + +- **GIVEN** a fresh HTTP bridge socket times out before any `response.*` event while using a proxy-injected durable anchor +- **AND** the exact anchor is quarantined successfully +- **WHEN** the client later resends self-contained full context without `previous_response_id` +- **THEN** the proxy verifies the retained input count and fingerprint plus safe-full-resend evidence +- **AND** does not re-inject the quarantined response id +- **AND** the full-context request is forwarded as an unanchored fresh response + +#### Scenario: Completed mid-tool full-history resend recovers after quarantine + +- **GIVEN** durable lookup identifies a quarantined latest-response anchor with retained input count and fingerprint +- **AND** the later unanchored request matches that stored prefix +- **WHEN** the projected suffix contains a complete supported direct tool call and its matching output +- **AND** the projected entire input has a self-contained call/output graph +- **AND** the projected suffix independently satisfies account-neutral fresh-input validation +- **THEN** the proxy forwards the full-context request as an unanchored fresh response +- **AND** recovery does not require a later assistant-final or user message +- **AND** generic and cross-account replay eligibility remain unchanged + +#### Scenario: Malformed mid-tool resend after quarantine fails closed + +- **GIVEN** durable lookup identifies a quarantined latest-response anchor +- **WHEN** a purported full-context resend has a mismatched prefix, orphan tool output, unresolved tool call, duplicate call id, or unsupported/account-scoped state in the new suffix +- **THEN** the proxy returns the full-resend-required HTTP 400 +- **AND** it does not create, forward, or submit an unanchored upstream request + +#### Scenario: Incremental request after quarantine fails closed + +- **GIVEN** durable lookup identifies a quarantined latest-response anchor +- **WHEN** the client sends incremental or prefix-mismatched input without an explicit `previous_response_id` +- **THEN** the proxy returns HTTP 400 with code `continuity_requires_full_resend`, type `invalid_request_error`, and parameter `input` +- **AND** it does not create, forward, or submit an unanchored upstream request + +#### Scenario: Repeated Goal continuation remains a deterministic client error + +- **GIVEN** an automatic durable anchor is quarantined or belongs to a prior socket +- **WHEN** a Goal client repeatedly submits the same incremental continuation without complete context +- **THEN** every attempt returns the same full-resend-required HTTP 400 +- **AND** the message requests complete context or a new session without claiming another upstream close +- **AND** no attempt creates an upstream transport or writes account health + +#### Scenario: Explicit conversation remains independent of automatic-anchor quarantine + +- **GIVEN** durable lookup identifies quarantined automatic response-anchor state +- **WHEN** the client supplies an explicit `conversation` without `previous_response_id` +- **THEN** the proxy does not reject the request solely because the automatic anchor is quarantined +- **AND** the existing explicit-conversation continuity path remains authoritative + +#### Scenario: Owner-forward refresh observes newly quarantined state + +- **GIVEN** an origin forwards a hard-continuity request using an earlier trusted owner lookup +- **WHEN** the forward fails before output and the refreshed lookup has an absent latest response id with retained input proof +- **AND** the request is incremental or unverifiable without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before local session creation or submission + +#### Scenario: Live recovery socket requires socket-local anchor provenance + +- **GIVEN** a live local recovery socket exists for a hard durable session +- **AND** that socket has not completed the durable latest response id with `store=false` +- **WHEN** the client sends incremental input without explicit response or conversation continuity +- **THEN** the proxy returns the full-resend-required HTTP 400 before submission +- **BUT WHEN** the client sends a fingerprint-verified self-contained full-history resend +- **THEN** the proxy may submit it unanchored on that socket + +#### Scenario: Socket replacement during gate wait cannot carry an automatic anchor + +- **GIVEN** a request serialized an automatic `store=false` anchor from the current WebSocket +- **AND** it waits for response-create admission +- **WHEN** the WebSocket lineage changes before the request is sent +- **THEN** the proxy does not submit the serialized anchor on the replacement socket +- **AND** a replay-safe full-history request may proceed without the anchor +- **AND** an anchor-dependent request receives the full-resend-required HTTP 400 before submission + +#### Scenario: Image-bearing full-history fallback remains upstream-safe + +- **GIVEN** a replay-safe full-history request has anchored and unanchored forms containing an external input-image URL +- **AND** HTTP bridge image inlining is enabled +- **WHEN** a socket replacement selects the unanchored form at the final send boundary +- **THEN** the selected frame retains the inlined image and contains no surviving external URL +- **AND** the selected transformed frame remains subject to the upstream serialized request-size guard + +#### Scenario: Timed-out incremental request is not replayed fresh + +- **GIVEN** a request depends on a proxy-injected durable anchor and is not independently self-contained +- **WHEN** it reaches the eventless missing-`response.created` deadline +- **THEN** the proxy returns the explicit terminal failure +- **AND** it does not retry that request without `previous_response_id` + +#### Scenario: First full-history retry after a store-false disconnect avoids the dead anchor + +- **GIVEN** an upstream WebSocket disconnects after a sent HTTP bridge request used a proxy-injected `store=false` anchor +- **WHEN** the client retries with verified self-contained full context +- **THEN** the closed socket's exact automatic anchor has already been quarantined +- **AND** the retry follows the existing unanchored quarantine-recovery guard +- **AND** it does not first wait for another missing-`response.created` deadline on the dead connection-local id + +#### Scenario: Escape interruption permits the next verified same-session turn + +- **GIVEN** the client cancels a downstream Codex SSE stream before its upstream response completes +- **AND** retiring the old socket conditionally quarantines its exact eligible automatic `store=false` anchor +- **WHEN** the client sends the next turn under the same session header with fingerprint-matched self-contained full history +- **THEN** the proxy opens a fresh upstream socket and submits that history without the quarantined anchor +- **AND** it does not reconnect or replay the interrupted request + +#### Scenario: Automatic encrypted compaction survives a fresh socket + +- **GIVEN** Codex replaces previously fingerprinted history with an upstream-issued encrypted `compaction` item +- **AND** durable hard-continuity state retains the original owner and prior input proof but no reusable socket +- **WHEN** Codex continues the same session without an explicit anchor +- **THEN** the service opens a fresh socket on that owner and forwards the compaction item unchanged without `previous_response_id` +- **AND** it does not classify the opaque context replacement as an incremental plaintext request + +#### Scenario: Idle close still invalidates the latest connection-local response + +- **GIVEN** the latest response completed with `store=false` on the current WebSocket +- **AND** the socket closes while no request is pending +- **WHEN** the client later sends verified self-contained full history +- **THEN** the completed response id has already been conditionally quarantined +- **AND** the retry starts a new unanchored lineage without a dead-anchor timeout + +#### Scenario: Process restart never reattaches a store-false durable id + +- **GIVEN** a process starts with a hard durable row whose latest response came from a previous WebSocket +- **AND** no live owner can receive the request on that socket +- **WHEN** the client sends a verified self-contained full-history request without an explicit anchor +- **THEN** the proxy forwards the request unanchored on the fresh socket +- **BUT WHEN** the client sends incremental or unverifiable input +- **THEN** the proxy returns the full-resend-required HTTP 400 before opening the upstream transport + +#### Scenario: Soft prompt-cache reconnect does not inherit hard continuity failure + +- **GIVEN** a soft prompt-cache bridge row has quarantined its closed socket's automatic `store=false` anchor +- **AND** the next request supplies no explicit `previous_response_id` +- **WHEN** a self-contained request reaches a fresh upstream socket +- **THEN** the proxy creates the soft-locality session without the quarantined id +- **AND** it does not apply the hard-continuity full-history guard solely because the soft row retains quarantine proof + ### Requirement: Codex WebSocket pre-created turns receive application heartbeats When serving the Codex-native `/backend-api/codex/responses` WebSocket route, the proxy SHALL emit a parseable Codex vendor heartbeat while a `response.create` request is pending but upstream has not yet emitted `response.created`. The heartbeat MUST be an application text frame so Codex clients reset stream-idle watchdogs that do not observe WebSocket protocol ping/pong frames. Once upstream assigns a response id, the proxy MUST continue using the existing `response.in_progress` heartbeat shape for that response id. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 989783c5af..faf4d8426f 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -527,6 +527,27 @@ async def send_text(self, text: str) -> None: ) +class _CompleteThenCreatedOnlyUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + if not self.sent_text: + await super().send_text(text) + return + self.sent_text.append(text) + response_id = f"resp_cancel_after_anchor_{len(self.sent_text)}" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": {"id": response_id, "object": "response", "status": "in_progress"}, + }, + separators=(",", ":"), + ), + ) + ) + + class _SilentUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): async def send_text(self, text: str) -> None: self.sent_text.append(text) @@ -13407,6 +13428,296 @@ async def fake_connect_responses_websocket( assert fake_upstream.closed is True +@pytest.mark.asyncio +async def test_backend_codex_escape_cancel_allows_next_same_session_full_history( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_escape_recovery", + "http-bridge-escape-recovery@example.com", + ) + service = get_proxy_service_for_app(app_instance) + account = await _get_account(account_id) + interrupted_upstream = _CompleteThenCreatedOnlyUpstreamWebSocket("resp_escape_source") + recovery_upstream = _FakeBridgeUpstreamWebSocket("resp_escape_recovered") + upstreams = [interrupted_upstream, recovery_upstream] + connect_count = 0 + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline, kwargs + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + upstream = upstreams[connect_count] + connect_count += 1 + return upstream + + account_health_write = AsyncMock() + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(service, "_handle_stream_error", account_health_write) + + session_id = "codex-escape-recovery" + session_headers = {"x-codex-session-id": session_id} + stored_input = [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "first question"}], + } + ] + initial_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": stored_input, + "stream": True, + }, + headers=session_headers, + ) + assert initial_events[-1]["type"] == "response.completed" + + tool_call = { + "type": "custom_tool_call", + "call_id": "call_escape", + "name": "shell", + "input": "pwd", + "status": "completed", + } + tool_output = { + "type": "custom_tool_call_output", + "call_id": "call_escape", + "output": "/workspace", + "status": "completed", + } + interrupted_input = [*stored_input, tool_call, tool_output] + interrupted_payload = proxy_module.ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": interrupted_input, + "stream": True, + } + ) + interrupted_stream = cast( + AsyncGenerator[str, None], + service._stream_via_http_bridge( + interrupted_payload, + session_headers, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=900.0, + max_sessions=128, + queue_limit=8, + ), + ) + created_block = await asyncio.wait_for(interrupted_stream.__anext__(), timeout=_TEST_SYNC_TIMEOUT_SECONDS) + assert "response.created" in created_block + + # Codex Escape closes the downstream stream while the response is still + # in progress. The bridge must fence the old connection-local anchor + # before retiring that socket. + await interrupted_stream.aclose() + durable_after_cancel = await service._durable_bridge.lookup_request_targets( + session_key_kind="session_header", + session_key_value=session_id, + api_key_id=None, + turn_state=None, + session_header=session_id, + previous_response_id=None, + ) + assert durable_after_cancel is not None + assert durable_after_cancel.latest_response_id is None + assert durable_after_cancel.latest_input_item_count == len(stored_input) + + followup_input = [ + *interrupted_input, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue after Escape"}], + }, + ] + recovered_events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": followup_input, + "stream": True, + }, + headers=session_headers, + ) + + assert recovered_events[-1]["type"] == "response.completed" + assert connect_count == 2 + assert interrupted_upstream.closed is True + assert len(interrupted_upstream.sent_text) == 2 + assert len(recovery_upstream.sent_text) == 1 + recovery_frame = json.loads(recovery_upstream.sent_text[0]) + assert "previous_response_id" not in recovery_frame + assert recovery_frame["input"] == followup_input + account_health_write.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_backend_codex_automatic_compaction_recovers_on_fresh_same_account_socket( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + owner_account_id = await _import_account( + async_client, + "acc_http_bridge_compaction_owner", + "http-bridge-compaction-owner@example.com", + ) + alternate_account_id = await _import_account( + async_client, + "acc_http_bridge_compaction_alternate", + "http-bridge-compaction-alternate@example.com", + ) + owner_account = await _get_account(owner_account_id) + alternate_account = await _get_account(alternate_account_id) + owner_chatgpt_account_id = cast(str, owner_account.chatgpt_account_id) + service = get_proxy_service_for_app(app_instance) + session_id = "codex-automatic-compaction-recovery" + old_input = [{"type": "message", "role": "user", "content": "the pre-compaction history"}] + old_fingerprint = proxy_module._fingerprint_input_items(old_input) + + claimed = await service._durable_bridge.claim_live_session( + session_key_kind="session_header", + session_key_value=session_id, + api_key_id=None, + instance_id="instance-a", + lease_ttl_seconds=60.0, + account_id=owner_account.id, + model="gpt-5.1", + service_tier=None, + latest_turn_state=session_id, + latest_response_id="resp_before_compaction", + allow_takeover=True, + ) + renewed = await service._durable_bridge.renew_live_session( + session_id=claimed.session_id, + api_key_id=None, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + lease_ttl_seconds=60.0, + latest_turn_state=session_id, + latest_response_id="resp_before_compaction", + latest_input_item_count=len(old_input), + latest_input_full_fingerprint=old_fingerprint, + ) + assert renewed is not None + released = await service._durable_bridge.release_live_session( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + draining=False, + ) + assert released is not None + assert released.latest_response_id == "resp_before_compaction" + assert released.latest_input_full_fingerprint == old_fingerprint + + selection_calls: list[dict[str, object]] = [] + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline + selection_calls.append(dict(kwargs)) + selected_account = ( + owner_account if kwargs.get("preferred_account_id") == owner_account.id else alternate_account + ) + return AccountSelection(account=selected_account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + upstream = _FakeBridgeUpstreamWebSocket("resp_compaction_recovered") + connected_account_ids: list[str] = [] + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, base_url, session + connected_account_ids.append(account_id_header) + return upstream + + account_health_write = AsyncMock() + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + monkeypatch.setattr(service, "_handle_stream_error", account_health_write) + + compaction_input = [ + { + "id": "cmp_automatic_recovery", + "type": "compaction", + "encrypted_content": "encrypted-complete-context", + }, + { + "type": "message", + "role": "user", + "content": "continue after automatic compaction", + }, + ] + events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": compaction_input, + "stream": True, + }, + headers={"x-codex-session-id": session_id}, + ) + + assert events[-1]["type"] == "response.completed" + assert connected_account_ids == [owner_chatgpt_account_id] + assert len(selection_calls) == 1 + assert selection_calls[0]["preferred_account_id"] == owner_account.id + assert selection_calls[0]["preferred_account_is_continuity_owner"] is True + assert selection_calls[0]["fallback_on_preferred_account_unavailable"] is False + assert len(upstream.sent_text) == 1 + recovery_frame = json.loads(upstream.sent_text[0]) + assert "previous_response_id" not in recovery_frame + assert recovery_frame["input"] == compaction_input + account_health_write.assert_not_awaited() + + @pytest.mark.asyncio async def test_prepare_http_bridge_request_preserves_existing_client_metadata(app_instance): service = get_proxy_service_for_app(app_instance) diff --git a/tests/unit/test_durable_bridge_sessions.py b/tests/unit/test_durable_bridge_sessions.py index 7d52d9376d..c838e2cbde 100644 --- a/tests/unit/test_durable_bridge_sessions.py +++ b/tests/unit/test_durable_bridge_sessions.py @@ -1485,6 +1485,264 @@ async def test_durable_bridge_claim_renews_same_owner_epoch( assert renewed.latest_response_id == "resp_2" +@pytest.mark.asyncio +async def test_durable_bridge_clears_expected_latest_response_anchor_and_retains_alias( + coordinator: DurableBridgeSessionCoordinator, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-quarantine-anchor", + api_key_id="key-quarantine-anchor", + instance_id="instance-a", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state="http_turn_quarantine", + latest_response_id=None, + allow_takeover=True, + ) + registered = await coordinator.register_previous_response_id( + session_id=claimed.session_id, + api_key_id="key-quarantine-anchor", + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + response_id="resp_poisoned", + lease_ttl_seconds=60.0, + input_item_count=17, + input_full_fingerprint="a" * 64, + pending_tool_calls={"call_pending": "custom_tool_call"}, + ) + assert registered == DurableBridgeAliasRegistration.REGISTERED + + cleared = await coordinator.clear_latest_response_anchor_if_current( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + expected_response_id="resp_poisoned", + ) + + assert cleared is not None + assert cleared.owner_instance_id == "instance-a" + assert cleared.owner_epoch == claimed.owner_epoch + assert cleared.latest_response_id is None + assert cleared.latest_input_item_count == 17 + assert cleared.latest_input_full_fingerprint == "a" * 64 + assert cleared.latest_pending_tool_calls is None + assert cleared.latest_response_anchor_quarantined is True + by_retained_alias = await coordinator.lookup_request_targets( + session_key_kind="request", + session_key_value="req-after-quarantine", + api_key_id="key-quarantine-anchor", + turn_state=None, + session_header=None, + previous_response_id="resp_poisoned", + ) + assert by_retained_alias is not None + assert by_retained_alias.session_id == claimed.session_id + assert by_retained_alias.latest_response_id is None + assert by_retained_alias.latest_input_item_count == 17 + assert by_retained_alias.latest_input_full_fingerprint == "a" * 64 + assert by_retained_alias.latest_response_anchor_quarantined is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("input_item_count", "input_full_fingerprint"), + [ + pytest.param(None, None, id="missing-proof"), + pytest.param(0, "", id="nonpositive-empty-proof"), + ], +) +async def test_durable_bridge_anchor_quarantine_without_prefix_proof_stays_fail_closed_until_replaced( + coordinator: DurableBridgeSessionCoordinator, + input_item_count: int | None, + input_full_fingerprint: str | None, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-quarantine-without-proof", + api_key_id=None, + instance_id="instance-a", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + registered = await coordinator.register_previous_response_id( + session_id=claimed.session_id, + api_key_id=None, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + response_id="resp_without_proof", + lease_ttl_seconds=60.0, + input_item_count=input_item_count, + input_full_fingerprint=input_full_fingerprint, + ) + assert registered == DurableBridgeAliasRegistration.REGISTERED + + cleared = await coordinator.clear_latest_response_anchor_if_current( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + expected_response_id="resp_without_proof", + ) + + assert cleared is not None + assert cleared.latest_response_id is None + assert cleared.latest_input_item_count is not None + assert cleared.latest_input_item_count < 0 + assert cleared.latest_input_full_fingerprint + assert cleared.latest_response_anchor_quarantined is True + by_retained_alias = await coordinator.lookup_request_targets( + session_key_kind="request", + session_key_value="req-after-proofless-quarantine", + api_key_id=None, + turn_state=None, + session_header=None, + previous_response_id="resp_without_proof", + ) + assert by_retained_alias is not None + assert by_retained_alias.session_id == claimed.session_id + assert by_retained_alias.latest_response_anchor_quarantined is True + + replacement_registered = await coordinator.register_previous_response_id( + session_id=claimed.session_id, + api_key_id=None, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + response_id="resp_with_real_proof", + lease_ttl_seconds=60.0, + input_item_count=4, + input_full_fingerprint="d" * 64, + ) + assert replacement_registered == DurableBridgeAliasRegistration.REGISTERED + replaced = await coordinator.lookup_request_targets( + session_key_kind="session_header", + session_key_value="sid-quarantine-without-proof", + api_key_id=None, + turn_state=None, + session_header=None, + previous_response_id=None, + ) + assert replaced is not None + assert replaced.latest_response_id == "resp_with_real_proof" + assert replaced.latest_input_item_count == 4 + assert replaced.latest_input_full_fingerprint == "d" * 64 + assert replaced.latest_response_anchor_quarantined is False + + +@pytest.mark.asyncio +async def test_durable_bridge_anchor_quarantine_preserves_concurrently_advanced_response( + coordinator: DurableBridgeSessionCoordinator, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-quarantine-newer-anchor", + api_key_id=None, + instance_id="instance-a", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + for response_id, fingerprint, pending_call in ( + ("resp_old", "a" * 64, "call_old"), + ("resp_new", "b" * 64, "call_new"), + ): + registered = await coordinator.register_previous_response_id( + session_id=claimed.session_id, + api_key_id=None, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + response_id=response_id, + lease_ttl_seconds=60.0, + input_item_count=9, + input_full_fingerprint=fingerprint, + pending_tool_calls={pending_call: "function_call"}, + ) + assert registered == DurableBridgeAliasRegistration.REGISTERED + + unchanged = await coordinator.clear_latest_response_anchor_if_current( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + expected_response_id="resp_old", + ) + + assert unchanged is not None + assert unchanged.latest_response_id == "resp_new" + assert unchanged.latest_input_item_count == 9 + assert unchanged.latest_input_full_fingerprint == "b" * 64 + assert unchanged.latest_pending_tool_calls == {"call_new": "function_call"} + + +@pytest.mark.asyncio +async def test_durable_bridge_anchor_quarantine_is_owner_epoch_fenced( + coordinator: DurableBridgeSessionCoordinator, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-quarantine-fenced", + api_key_id=None, + instance_id="instance-a", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + registered = await coordinator.register_previous_response_id( + session_id=claimed.session_id, + api_key_id=None, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + response_id="resp_owned_by_replacement", + lease_ttl_seconds=60.0, + input_item_count=5, + input_full_fingerprint="c" * 64, + pending_tool_calls={"call_replacement": "apply_patch_call"}, + ) + assert registered == DurableBridgeAliasRegistration.REGISTERED + replacement = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-quarantine-fenced", + api_key_id=None, + instance_id="instance-b", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + + unchanged = await coordinator.clear_latest_response_anchor_if_current( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + expected_response_id="resp_owned_by_replacement", + ) + + assert replacement.owner_epoch == claimed.owner_epoch + 1 + assert unchanged is not None + assert unchanged.owner_instance_id == "instance-b" + assert unchanged.owner_epoch == replacement.owner_epoch + assert unchanged.latest_response_id == "resp_owned_by_replacement" + assert unchanged.latest_input_item_count == 5 + assert unchanged.latest_input_full_fingerprint == "c" * 64 + assert unchanged.latest_pending_tool_calls == {"call_replacement": "apply_patch_call"} + + @pytest.mark.asyncio async def test_durable_bridge_account_change_advances_epoch_to_fence_stale_release( coordinator: DurableBridgeSessionCoordinator, diff --git a/tests/unit/test_http_bridge_cancel_drain.py b/tests/unit/test_http_bridge_cancel_drain.py index 2ba47a109e..04583e4ace 100644 --- a/tests/unit/test_http_bridge_cancel_drain.py +++ b/tests/unit/test_http_bridge_cancel_drain.py @@ -14,6 +14,7 @@ from app.db.models import AccountStatus, Base from app.modules.api_keys.service import ApiKeyData, ApiKeyUsageReservationData from app.modules.proxy import service as proxy_service +from app.modules.proxy._service.http_bridge import request_submit as http_bridge_request_submit_module from app.modules.proxy.durable_bridge_coordinator import DurableBridgeSessionCoordinator pytestmark = pytest.mark.unit @@ -170,6 +171,160 @@ async def test_cancelled_http_bridge_request_retires_session_before_retry_overla release_reservation.assert_awaited_once_with(cancelled_request.api_key_reservation) +@pytest.mark.asyncio +async def test_cancelled_http_bridge_request_quarantines_socket_anchor_before_durable_release() -> None: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + coordinator = DurableBridgeSessionCoordinator(cast(Callable[[], AsyncSession], session_factory)) + instance_id = proxy_service.get_settings().http_responses_session_bridge_instance_id + anchor_id = "resp-cancel-anchor" + input_fingerprint = proxy_service._fingerprint_input_items([{"role": "user", "content": "first question"}]) + lookup = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-cancel-drain", + api_key_id=None, + instance_id=instance_id, + lease_ttl_seconds=60.0, + account_id="acc-cancel-drain", + model="gpt-5.5", + service_tier=None, + latest_turn_state="turn-cancel-drain", + latest_response_id=anchor_id, + allow_takeover=True, + ) + refreshed = await coordinator.renew_live_session( + session_id=lookup.session_id, + api_key_id=None, + instance_id=instance_id, + owner_epoch=lookup.owner_epoch, + lease_ttl_seconds=60.0, + latest_response_id=anchor_id, + latest_input_item_count=1, + latest_input_full_fingerprint=input_fingerprint, + latest_pending_tool_calls={"call-cancel": "custom_tool_call"}, + ) + assert refreshed is not None + + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + service._durable_bridge = coordinator # noqa: SLF001 + cancelled_request = _make_request_state( + "req-cancelled-anchor", + response_id="resp-cancelled-in-progress", + awaiting_response_created=False, + event_queue=asyncio.Queue(), + ) + cancelled_request.skip_request_log = False + cancelled_request.response_create_sent_at = 2.0 + cancelled_request.response_store = False + cancelled_request.proxy_injected_previous_response_id = True + cancelled_request.previous_response_id = anchor_id + session = _make_http_bridge_session(deque([cancelled_request]), queued_request_count=1) + session.durable_session_id = lookup.session_id + session.durable_owner_epoch = lookup.owner_epoch + session.last_completed_response_id = anchor_id + session.last_completed_response_store = False + session.last_completed_input_count = 1 + session.last_completed_input_prefix_fingerprint = input_fingerprint + session.last_pending_tool_calls = {"call-cancel": "custom_tool_call"} + + try: + detached = await service._detach_http_bridge_request(session, request_state=cancelled_request) + durable_after_cancel = await coordinator.lookup_request_targets( + session_key_kind="session_header", + session_key_value="sid-cancel-drain", + api_key_id=None, + turn_state=None, + session_header="sid-cancel-drain", + previous_response_id=None, + ) + finally: + await engine.dispose() + + assert detached is True + assert durable_after_cancel is not None + assert durable_after_cancel.latest_response_id is None + assert durable_after_cancel.latest_input_item_count == 1 + assert durable_after_cancel.latest_input_full_fingerprint == input_fingerprint + assert durable_after_cancel.latest_pending_tool_calls is None + assert session.last_completed_response_id is None + assert session.last_completed_response_store is None + assert session.last_pending_tool_calls == {} + assert session.closed is True + cast(Any, session.upstream).close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_cancelled_http_bridge_request_fences_quarantine_through_socket_retirement( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, SimpleNamespace())) + cancelled_request = _make_request_state( + "req-cancelled-race", + response_id="resp-cancelled-race", + awaiting_response_created=False, + event_queue=asyncio.Queue(), + ) + cancelled_request.skip_request_log = False + cancelled_request.response_create_sent_at = 2.0 + cancelled_request.response_store = False + cancelled_request.proxy_injected_previous_response_id = True + cancelled_request.previous_response_id = "resp-anchor-race" + session = _make_http_bridge_session(deque([cancelled_request]), queued_request_count=1) + session.last_completed_response_id = "resp-anchor-race" + session.last_completed_response_store = False + + quarantine_started = asyncio.Event() + allow_quarantine = asyncio.Event() + lifecycle_order: list[str] = [] + + async def quarantine_anchor( + _service: object, + _session: proxy_service._HTTPBridgeSession, + *, + expected_response_id: str, + lifecycle_lock_held: bool = False, + ) -> bool: + assert expected_response_id == "resp-anchor-race" + assert lifecycle_lock_held is True + lifecycle_order.append("quarantine_started") + quarantine_started.set() + await allow_quarantine.wait() + lifecycle_order.append("quarantine_finished") + return True + + async def close_session( + _session: proxy_service._HTTPBridgeSession, + *, + reason: str, + ) -> None: + assert reason == "retire_after_drain" + lifecycle_order.append("socket_retired") + _session.closed = True + + monkeypatch.setattr( + http_bridge_request_submit_module, + "_quarantine_http_bridge_disconnected_socket_anchor", + quarantine_anchor, + ) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", close_session) + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", AsyncMock()) + + detach_task = asyncio.create_task(service._detach_http_bridge_request(session, request_state=cancelled_request)) + await asyncio.wait_for(quarantine_started.wait(), timeout=1.0) + competing_retire = asyncio.create_task(service._retire_http_bridge_after_drain_if_ready(session)) + await asyncio.sleep(0) + + assert lifecycle_order == ["quarantine_started"] + + allow_quarantine.set() + detached, _ = await asyncio.gather(detach_task, competing_retire) + + assert detached is True + assert lifecycle_order == ["quarantine_started", "quarantine_finished", "socket_retired"] + + def test_retiring_http_bridge_session_is_not_reusable() -> None: session = _make_http_bridge_session(deque(), queued_request_count=0) session.upstream_control.retire_after_drain = True diff --git a/tests/unit/test_http_bridge_forwarding.py b/tests/unit/test_http_bridge_forwarding.py index 983761c7ec..ebc5fbe26b 100644 --- a/tests/unit/test_http_bridge_forwarding.py +++ b/tests/unit/test_http_bridge_forwarding.py @@ -21,6 +21,7 @@ HTTP_BRIDGE_FORWARDED_HEADER, HTTP_BRIDGE_ORIGIN_INSTANCE_HEADER, HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER, + HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER, HTTP_BRIDGE_RESERVATION_ID_HEADER, HTTP_BRIDGE_RESERVATION_KEY_ID_HEADER, HTTP_BRIDGE_RESERVATION_MODEL_HEADER, @@ -122,6 +123,81 @@ def test_parse_forwarded_request_accepts_signed_internal_forward() -> None: assert forwarded.context.original_affinity_key is None +def test_parse_forwarded_request_preserves_signed_proxy_injected_anchor_provenance() -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": "follow-up", + "previous_response_id": "resp_proxy_injected", + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=True, + downstream_turn_state="http_turn_123", + proxy_injected_previous_response_id=True, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + + forwarded, error = parse_forwarded_request( + headers, + payload=payload, + current_instance="instance-b", + ) + + assert headers[HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER] == "1" + assert headers[HTTP_BRIDGE_SIGNATURE_VERSION_HEADER] == "2" + assert headers[HTTP_BRIDGE_ORIGINAL_UNANCHORED_HEADER] == "0" + assert error is None + assert forwarded is not None + assert forwarded.context.proxy_injected_previous_response_id is True + assert forwarded.context.signature_version == "2" + + +@pytest.mark.parametrize("tamper", ["add", "add_false", "change", "strip", "strip_full_signature"]) +def test_parse_forwarded_request_rejects_tampered_proxy_injected_anchor_provenance( + tamper: str, +) -> None: + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": "follow-up", + "previous_response_id": "resp_proxy_injected", + } + ) + context = HTTPBridgeForwardContext( + origin_instance="instance-a", + target_instance="instance-b", + codex_session_affinity=True, + downstream_turn_state="http_turn_123", + proxy_injected_previous_response_id=tamper not in {"add", "add_false"}, + ) + headers = build_owner_forward_headers(headers={}, payload=payload, context=context) + if tamper == "add": + headers[HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER] = "1" + elif tamper == "add_false": + headers[HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER] = "0" + elif tamper == "change": + headers[HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER] = "0" + elif tamper == "strip": + headers.pop(HTTP_BRIDGE_PROXY_INJECTED_PREVIOUS_RESPONSE_HEADER) + else: + headers.pop(HTTP_BRIDGE_SIGNATURE_V2_HEADER) + + forwarded, error = parse_forwarded_request( + headers, + payload=payload, + current_instance="instance-b", + ) + + assert forwarded is None + assert error is not None + assert error.payload["error"]["code"] == "bridge_forward_invalid" + + def test_parse_forwarded_request_preserves_signed_file_owner_proof() -> None: payload = _payload() context = HTTPBridgeForwardContext( diff --git a/tests/unit/test_network_recovery.py b/tests/unit/test_network_recovery.py index 130ad66180..a1f08111ca 100644 --- a/tests/unit/test_network_recovery.py +++ b/tests/unit/test_network_recovery.py @@ -4,6 +4,7 @@ import errno import logging import socket +import threading from typing import cast from unittest.mock import AsyncMock @@ -66,6 +67,177 @@ def test_process_network_error_requires_stable_code_not_message_text() -> None: assert not network_recovery.is_process_network_error("upstream_unavailable") +@pytest.mark.asyncio +async def test_websocket_egress_failure_correlator_marks_all_cross_account_waiters_and_trailing_failure() -> None: + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.2, + max_observations=16, + ) + + first = asyncio.create_task( + correlator.observe( + egress_key="environment_proxy:http://proxy.example:8080", + account_id="acc-a", + ) + ) + await asyncio.sleep(0) + second = asyncio.create_task( + correlator.observe( + egress_key="environment_proxy:http://proxy.example:8080", + account_id="acc-b", + ) + ) + + assert await asyncio.gather(first, second) == [True, True] + assert ( + await asyncio.wait_for( + correlator.observe( + egress_key="environment_proxy:http://proxy.example:8080", + account_id="acc-a", + ), + timeout=0.05, + ) + is True + ) + + +@pytest.mark.asyncio +async def test_websocket_egress_failure_correlator_records_nonwaiting_cross_account_evidence() -> None: + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.2, + max_observations=16, + ) + ambiguous = asyncio.create_task( + correlator.observe( + egress_key="direct:wss://chatgpt.com:443", + account_id="acc-a", + ) + ) + await asyncio.sleep(0) + + known_neutral = await asyncio.wait_for( + correlator.observe( + egress_key="direct:wss://chatgpt.com:443", + account_id="acc-b", + wait_for_correlation=False, + ), + timeout=0.05, + ) + + assert known_neutral is True + assert await asyncio.wait_for(ambiguous, timeout=0.05) is True + + +@pytest.mark.asyncio +async def test_websocket_egress_failure_correlator_rejects_same_account_different_egress_and_anonymous() -> None: + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.02, + max_observations=16, + ) + + same_account = await asyncio.gather( + correlator.observe(egress_key="direct:wss://chatgpt.com:443", account_id="acc-a"), + correlator.observe(egress_key="direct:wss://chatgpt.com:443", account_id="acc-a"), + ) + different_egress = await asyncio.gather( + correlator.observe(egress_key="routed_proxy:ep-a", account_id="acc-a"), + correlator.observe(egress_key="routed_proxy:ep-b", account_id="acc-b"), + ) + anonymous = await asyncio.wait_for( + correlator.observe(egress_key="direct:wss://chatgpt.com:443", account_id=None), + timeout=0.01, + ) + + assert same_account == [False, False] + assert different_egress == [False, False] + assert anonymous is False + + +@pytest.mark.asyncio +async def test_websocket_egress_failure_correlator_retains_cancelled_observation_without_waiter() -> None: + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.2, + max_observations=16, + ) + cancelled = asyncio.create_task( + correlator.observe( + egress_key="environment_proxy:http://proxy.example:8081", + account_id="acc-a", + ) + ) + await asyncio.sleep(0) + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled + + assert ( + await asyncio.wait_for( + correlator.observe( + egress_key="environment_proxy:http://proxy.example:8081", + account_id="acc-b", + ), + timeout=0.05, + ) + is True + ) + + +@pytest.mark.asyncio +async def test_websocket_egress_failure_correlator_evicts_oldest_observation_at_capacity() -> None: + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.03, + max_observations=2, + ) + + for egress_key, account_id in ( + ("routed_proxy:ep-oldest", "acc-a"), + ("routed_proxy:ep-middle", "acc-c"), + ("routed_proxy:ep-newest", "acc-d"), + ): + candidate = asyncio.create_task( + correlator.observe( + egress_key=egress_key, + account_id=account_id, + ) + ) + await asyncio.sleep(0) + candidate.cancel() + with pytest.raises(asyncio.CancelledError): + await candidate + + assert ( + await correlator.observe( + egress_key="routed_proxy:ep-oldest", + account_id="acc-b", + ) + is False + ) + + +@pytest.mark.asyncio +async def test_websocket_egress_failure_correlator_notifies_waiters_across_event_loops() -> None: + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.5, + max_observations=16, + ) + ready = threading.Barrier(2) + + def observe_in_thread(account_id: str) -> bool: + async def observe() -> bool: + ready.wait() + return await correlator.observe( + egress_key="environment_proxy:http://proxy.cross-loop.test:8080", + account_id=account_id, + ) + + return asyncio.run(observe()) + + assert await asyncio.gather( + asyncio.to_thread(observe_in_thread, "acc-a"), + asyncio.to_thread(observe_in_thread, "acc-b"), + ) == [True, True] + + @pytest.mark.asyncio async def test_recovery_controller_retries_and_logs_recovery(monkeypatch, caplog) -> None: sleep = AsyncMock() diff --git a/tests/unit/test_proxy_api_responses_contract.py b/tests/unit/test_proxy_api_responses_contract.py index 5694f420cf..dd2eff19b6 100644 --- a/tests/unit/test_proxy_api_responses_contract.py +++ b/tests/unit/test_proxy_api_responses_contract.py @@ -1367,8 +1367,14 @@ async def test_normalize_public_responses_stream_codex_route_does_not_duplicate_ @pytest.mark.asyncio +@pytest.mark.parametrize( + ("original_request_unanchored", "proxy_injected_previous_response_id"), + [(True, False), (False, True)], +) async def test_internal_bridge_responses_disables_openai_sdk_contract( monkeypatch: pytest.MonkeyPatch, + original_request_unanchored: bool, + proxy_injected_previous_response_id: bool, ) -> None: from unittest.mock import AsyncMock @@ -1389,7 +1395,8 @@ async def fake_stream_responses(*args: object, **kwargs: object) -> object: target_instance="owner-b", codex_session_affinity=True, downstream_turn_state="http_turn_generated", - original_request_unanchored=True, + original_request_unanchored=original_request_unanchored, + proxy_injected_previous_response_id=proxy_injected_previous_response_id, original_affinity_kind="session", original_affinity_key="sid-abc", reservation=None, @@ -1413,7 +1420,12 @@ def fake_parse(headers, *, payload, current_instance): # Minimal payload + request stubs. from app.core.openai.requests import ResponsesRequest - payload = ResponsesRequest(model="gpt-5.5", input="hi", instructions="") + payload = ResponsesRequest( + model="gpt-5.5", + input="hi", + instructions="", + previous_response_id="resp_proxy_injected" if proxy_injected_previous_response_id else None, + ) class _StubRequest: @property @@ -1439,11 +1451,12 @@ def headers(self) -> dict[str, str]: f"internal_bridge_responses must pass enforce_openai_sdk_contract=False; got kwargs={kwargs!r}" ) assert kwargs.get("forwarded_downstream_turn_state") == "http_turn_generated" - assert kwargs.get("forwarded_original_request_unanchored") is True + assert kwargs.get("forwarded_original_request_unanchored") is original_request_unanchored + assert kwargs.get("forwarded_proxy_injected_previous_response_id") is proxy_injected_previous_response_id assert kwargs.get("forwarded_legacy_signature") is False forwarded_headers = kwargs.get("forwarded_headers") assert isinstance(forwarded_headers, dict) - assert "x-codex-turn-state" not in forwarded_headers + assert ("x-codex-turn-state" not in forwarded_headers) is original_request_unanchored @pytest.mark.asyncio diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 00dd232e8e..a3140bba43 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -17,9 +17,13 @@ import anyio import pytest from fastapi import WebSocket +from fastapi.responses import StreamingResponse +from starlette.requests import Request from websockets.exceptions import ConnectionClosedError from websockets.frames import Close +import app.core.clients.proxy_websocket as proxy_websocket_module +import app.core.resilience.network_recovery as network_recovery from app.core.auth.refresh import RefreshError from app.core.clients.proxy import CODEX_RESPONSES_LITE_WEBSOCKET_METADATA_KEY, ProxyResponseError from app.core.clients.proxy_websocket import ( @@ -33,6 +37,8 @@ from app.core.errors import openai_error from app.core.utils.request_id import get_request_id, reset_request_scope_id, set_request_scope_id from app.db.models import AccountStatus, HttpBridgeSessionState +from app.dependencies import ProxyContext +from app.modules.proxy import api as proxy_api_module from app.modules.proxy import http_bridge_forwarding as http_bridge_forwarding_module from app.modules.proxy import service as proxy_service from app.modules.proxy._service import support as proxy_support_module @@ -79,6 +85,24 @@ def _make_app_settings(*, bridge_enabled: bool = True, **overrides: Any) -> Sett return Settings(http_responses_session_bridge_enabled=bridge_enabled, **overrides) +_HTTP_BRIDGE_FULL_RESEND_REQUIRED_PAYLOAD = { + "error": { + "message": ( + "HTTP bridge continuity cannot resume incrementally. " + "Resend the complete conversation context in input or create a new session." + ), + "type": "invalid_request_error", + "code": "continuity_requires_full_resend", + "param": "input", + } +} + + +def _assert_http_bridge_full_resend_required(exc: ProxyResponseError) -> None: + assert exc.status_code == 400 + assert exc.payload == _HTTP_BRIDGE_FULL_RESEND_REQUIRED_PAYLOAD + + def _make_bridge_session( *, key: proxy_service._HTTPBridgeSessionKey | None = None, @@ -3241,6 +3265,7 @@ async def test_http_bridge_precreated_completed_terminal_falls_back_to_unresolve awaiting_response_created=True, event_queue=asyncio.Queue(), transport="http", + response_store=False, skip_request_log=True, ) session = _make_bridge_session( @@ -3294,6 +3319,7 @@ async def test_http_bridge_precreated_completed_terminal_falls_back_to_unresolve ] assert request_state.response_id == "resp_precreated_completed" assert session.last_completed_response_id == "resp_precreated_completed" + assert session.last_completed_response_store is False assert session.queued_request_count == 0 assert not session.pending_requests register_previous.assert_awaited_once() @@ -4252,6 +4278,31 @@ def test_http_bridge_request_text_replaces_client_installation_id() -> None: } +@pytest.mark.parametrize("requested_store", [False, True]) +def test_http_bridge_request_state_records_effective_response_store(requested_store: bool) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "", + "input": "hello", + "store": requested_store, + } + ) + + request_state, text_data = service._prepare_http_bridge_request( + payload, + {}, + api_key=None, + api_key_reservation=None, + ) + + # ResponsesRequest enforces ZDR for this surface, so provenance must + # reflect the effective upstream value rather than the raw client value. + assert request_state.response_store is False + assert json.loads(text_data)["store"] is False + + def test_http_bridge_request_text_rejects_installation_metadata_size_overflow( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -5940,6 +5991,8 @@ async def test_reconnect_http_bridge_session_filters_http_headers_for_upstream_w "x-handshake-debug": "1", } session.upstream_turn_state = "upstream-turn-state" + session.last_completed_response_id = "resp-before-reconnect" + session.last_completed_response_store = False captured_headers: list[dict[str, str]] = [] async def select_account(_deadline: float, **_: object) -> proxy_service.AccountSelection: @@ -5980,6 +6033,8 @@ async def open_upstream(_account: object, headers: dict[str, str], **_: object) await service._reconnect_http_bridge_session(session, request_state=request_state) assert captured_headers + assert session.last_completed_response_id == "resp-before-reconnect" + assert session.last_completed_response_store is None forwarded = {key.lower(): value for key, value in captured_headers[0].items()} assert forwarded["session_id"] == "sid-filtered" assert forwarded["user-agent"] == "pi" @@ -6655,7 +6710,7 @@ def test_durable_bridge_lookup_active_owner_accepts_naive_datetime() -> None: @pytest.mark.asyncio -async def test_stream_via_http_bridge_injects_durable_previous_response_anchor( +async def test_stream_via_http_bridge_rejects_incremental_durable_anchor_on_fresh_socket( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) @@ -6747,13 +6802,13 @@ def fake_prepare( ), ) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + get_or_create = AsyncMock(return_value=session) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( + with pytest.raises(ProxyResponseError) as exc_info: + async for _chunk in service._stream_via_http_bridge( payload, headers={"x-codex-session-id": "sid-123"}, codex_session_affinity=True, @@ -6766,11 +6821,12 @@ def fake_prepare( codex_idle_ttl_seconds=1800.0, max_sessions=8, queue_limit=4, - ) - ] + ): + pass - assert chunks == [] - assert captured["previous_response_id"] == "resp_latest" + _assert_http_bridge_full_resend_required(exc_info.value) + assert captured["previous_response_id"] is None + get_or_create.assert_not_awaited() @pytest.mark.asyncio @@ -6877,7 +6933,8 @@ def fake_prepare( monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + get_or_create = AsyncMock(return_value=session) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) @@ -6900,6 +6957,10 @@ def fake_prepare( ] assert chunks == [] + assert request_state.previous_response_id == "resp_prev_tool_call" + creation = get_or_create.await_args + assert creation is not None + assert creation.kwargs["previous_response_id"] == "resp_prev_tool_call" assert captured_input == [ { "type": "function_call_output", @@ -7138,17 +7199,147 @@ def fake_prepare( assert prepared_previous_response_ids == [None] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("latest_response_store", "expected_previous_response_ids"), + [ + pytest.param(False, [None, "resp-current-socket", "resp-current-socket"], id="store-false"), + pytest.param(None, [None], id="unknown-socket-provenance"), + pytest.param(True, [None], id="persisted-response"), + ], +) +async def test_stream_via_http_bridge_injects_only_current_socket_store_false_session_anchor( + monkeypatch: pytest.MonkeyPatch, + latest_response_store: bool | None, + expected_previous_response_ids: list[str | None], +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + stored_input: list[proxy_service.JsonValue] = [{"role": "user", "content": "first"}] + input_items: list[proxy_service.JsonValue] = [ + *stored_input, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "answer"}], + }, + {"role": "user", "content": "follow up"}, + ] + payload = proxy_service.ResponsesRequest.model_validate( + {"model": "gpt-5.4", "instructions": "hi", "input": input_items}, + ) + prepared_previous_response_ids: list[str | None] = [] + prepared_inputs: list[proxy_service.JsonValue] = [] + prepared_states: list[proxy_service._WebSocketRequestState] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del _headers, api_key, api_key_reservation, request_id, client_ip + prepared_previous_response_ids.append(prepared_payload.previous_response_id) + prepared_inputs.append(prepared_payload.input) + state = proxy_service._WebSocketRequestState( + request_id=f"req-current-socket-{len(prepared_states)}", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + previous_response_id=prepared_payload.previous_response_id, + transport="http", + ) + prepared_states.append(state) + return state, json.dumps( + { + "type": "response.create", + "previous_response_id": prepared_payload.previous_response_id, + "input": prepared_payload.input, + }, + separators=(",", ":"), + ) + + session = _make_bridge_session(key_value="sid-current-socket") + session.codex_session = True + session.last_completed_response_id = "resp-current-socket" + session.last_completed_response_store = latest_response_store + session.last_completed_input_count = len(stored_input) + session.last_completed_input_prefix_fingerprint = proxy_service._fingerprint_input_items(stored_input) + + async def stream_session_events(*_args: object, **_kwargs: object): + if False: + yield "" + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", stream_session_events) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-current-socket"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == [] + assert prepared_previous_response_ids == expected_previous_response_ids + if latest_response_store is False: + assert prepared_inputs[-1] == input_items[len(stored_input) :] + assert prepared_states[-1].proxy_injected_previous_response_id is True + else: + assert prepared_inputs == [input_items] + assert prepared_states[-1].proxy_injected_previous_response_id is False + + @pytest.mark.asyncio async def test_stream_via_http_bridge_does_not_inject_durable_previous_response_anchor_for_full_resend_payload( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) + stored_input: list[proxy_service.JsonValue] = [{"role": "user", "content": "hello"}] payload = proxy_service.ResponsesRequest.model_validate( { "model": "gpt-5.4", "instructions": "hi", "input": [ - {"role": "user", "content": "hello"}, + *stored_input, {"role": "assistant", "content": "world"}, {"role": "user", "content": "follow up"}, ], @@ -7238,6 +7429,8 @@ def fake_prepare( state=HttpBridgeSessionState.ACTIVE, latest_turn_state="http_turn_1", latest_response_id="resp_latest", + latest_input_item_count=len(stored_input), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input), ) ), ) @@ -7276,7 +7469,13 @@ def fake_prepare( @pytest.mark.asyncio @pytest.mark.parametrize( - ("suffix_items", "pending_tool_calls", "preserves_full_resend", "forwardable_owner"), + ( + "suffix_items", + "pending_tool_calls", + "expected_allowed", + "forwardable_owner", + "live_local_session_response_id", + ), [ pytest.param( [ @@ -7286,6 +7485,7 @@ def fake_prepare( None, True, False, + None, id="retained-assistant-output", ), pytest.param( @@ -7305,6 +7505,7 @@ def fake_prepare( {"call-1": "function_call"}, True, False, + None, id="self-contained-tool-loop", ), pytest.param( @@ -7322,15 +7523,57 @@ def fake_prepare( }, ], None, + True, + False, + None, + id="self-contained-tool-loop-without-manifest", + ), + pytest.param( + [ + { + "type": "function_call", + "call_id": "call-1", + "name": "lookup", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call-1", + "output": "result", + }, + ], + None, + True, False, + "resp_latest", + id="live-socket-mid-tool-retry-propagates-full-resend-safety", + ), + pytest.param( + [ + { + "type": "function_call", + "call_id": "call-1", + "name": "lookup", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call-1", + "output": "result", + }, + ], + None, + True, False, - id="tool-loop-with-unknown-manifest", + "resp-stale-current-socket", + id="live-socket-stale-provenance-does-not-bind-durable-anchor", ), pytest.param( [{"role": "user", "content": "revise that answer"}], None, False, False, + None, id="missing-prior-output", ), pytest.param( @@ -7338,16 +7581,18 @@ def fake_prepare( None, False, True, + None, id="owner-forward-race-missing-prior-output", ), ], ) -async def test_stream_via_http_bridge_preserves_only_safe_trimmable_full_resend_on_fresh_bridge( +async def test_stream_via_http_bridge_allows_only_safe_trimmable_full_resend_on_fresh_bridge( monkeypatch: pytest.MonkeyPatch, suffix_items: list[proxy_service.JsonValue], pending_tool_calls: dict[str, str] | None, - preserves_full_resend: bool, + expected_allowed: bool, forwardable_owner: bool, + live_local_session_response_id: str | None, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) stored_input_items: list[proxy_service.JsonValue] = [ @@ -7480,6 +7725,10 @@ def fake_prepare( ), ) session.codex_session = True + live_local_session_exists = live_local_session_response_id is not None + if live_local_session_exists: + session.last_completed_response_id = live_local_session_response_id + session.last_completed_response_store = False account_neutral_classifier = Mock(return_value=True) monkeypatch.setattr( http_bridge_streaming_module, @@ -7492,41 +7741,68 @@ def fake_prepare( "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=forwardable_owner), ) + monkeypatch.setattr( + service, + "_http_bridge_has_live_local_session", + AsyncMock(return_value=live_local_session_exists), + ) monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) get_or_create = AsyncMock(return_value=session) monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"x-codex-session-id": "sid-123"}, - codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ) - ] + async def collect_chunks() -> list[str]: + return [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-123"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + if not expected_allowed: + with pytest.raises(ProxyResponseError) as exc_info: + await collect_chunks() + _assert_http_bridge_full_resend_required(exc_info.value) + assert prepared_previous_response_ids == [None] + assert prepared_input_lengths == [len(input_items)] + assert all("previous_response_id" not in frame for frame in prepared_frames) + if forwardable_owner: + get_or_create.assert_awaited_once() + else: + get_or_create.assert_not_awaited() + account_neutral_classifier.assert_not_called() + return + + chunks = await collect_chunks() assert chunks == [] - assert prepared_previous_response_ids == ([None] if preserves_full_resend else [None, "resp_latest", "resp_latest"]) - assert prepared_input_lengths == ( - [len(input_items)] if preserves_full_resend else [len(input_items), len(input_items), len(suffix_items)] + matching_current_socket_anchor = live_local_session_response_id == "resp_latest" + expected_prepared_previous_response_ids = ( + [None, "resp_latest", "resp_latest"] if matching_current_socket_anchor else [None] + ) + assert prepared_previous_response_ids == expected_prepared_previous_response_ids + expected_prepared_input_lengths = ( + [len(input_items), len(input_items), len(suffix_items)] + if matching_current_socket_anchor + else [len(input_items)] ) + assert prepared_input_lengths == expected_prepared_input_lengths assert all("tools" not in frame for frame in prepared_frames) normalized_input_items = cast(list[proxy_service.JsonValue], payload.input) - expected_input_items = ( - normalized_input_items if preserves_full_resend else normalized_input_items[-len(suffix_items) :] - ) - assert prepared_frames[-1]["input"] == expected_input_items + expected_submitted_input = suffix_items if matching_current_socket_anchor else normalized_input_items + assert prepared_frames[-1]["input"] == expected_submitted_input assert [frame["client_metadata"][CODEX_RESPONSES_LITE_WEBSOCKET_METADATA_KEY] for frame in prepared_frames] == [ "true", ] * len(prepared_frames) @@ -7543,20 +7819,23 @@ def fake_prepare( assert cast(dict[str, Any], payload.to_payload()["reasoning"])["context"] == "last_turn" creation = get_or_create.await_args assert creation is not None - assert creation.kwargs["previous_response_id"] == ( - None if preserves_full_resend or forwardable_owner else "resp_latest" - ) + assert creation.kwargs["previous_response_id"] is None assert creation.kwargs["preferred_account_id"] == "acc-1" - assert session.last_completed_response_id == (None if preserves_full_resend else "resp_latest") - if not preserves_full_resend: - assert request_state.proxy_injected_previous_response_id is True - assert request_state.fresh_upstream_request_is_retry_safe is False + expected_session_response_id = "resp_latest" if live_local_session_exists else None + assert session.last_completed_response_id == expected_session_response_id + expected_session_response_store = False if matching_current_socket_anchor else None + assert session.last_completed_response_store is expected_session_response_store + assert request_state.proxy_injected_previous_response_id is matching_current_socket_anchor + if matching_current_socket_anchor: + assert request_state.fresh_upstream_request_is_retry_safe is True account_neutral_classifier.assert_not_called() @pytest.mark.asyncio +@pytest.mark.parametrize("quarantined", [False, True], ids=["retained-anchor", "quarantined-anchor"]) async def test_stream_via_http_bridge_does_not_inject_durable_previous_response_anchor_for_explicit_prompt_cache_key( monkeypatch: pytest.MonkeyPatch, + quarantined: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) payload = proxy_service.ResponsesRequest.model_validate( @@ -7647,7 +7926,9 @@ def fake_prepare( lease_expires_at=datetime.now(timezone.utc), state=HttpBridgeSessionState.ACTIVE, latest_turn_state="http_turn_1", - latest_response_id="resp_latest", + latest_response_id=None if quarantined else "resp_latest", + latest_input_item_count=1 if quarantined else None, + latest_input_full_fingerprint="soft-prefix-proof" if quarantined else None, ) ), ) @@ -7679,30 +7960,547 @@ def fake_prepare( @pytest.mark.asyncio -@pytest.mark.parametrize("stored_model", [None, "gpt-5.3"]) -async def test_stream_via_http_bridge_does_not_prefer_durable_account_for_soft_prompt_cache_lookup( +@pytest.mark.parametrize( + ("request_kind", "expected_allowed"), + [ + pytest.param("completed_response", True, id="completed-response-and-new-user"), + pytest.param("mid_tool_with_user", True, id="observed-mid-tool-history-with-user"), + pytest.param("mid_tool_without_user", True, id="immediate-mid-tool-retry-without-user"), + pytest.param("explicit_conversation", True, id="explicit-conversation"), + pytest.param("incremental", False, id="incremental"), + pytest.param("fingerprint_mismatch", False, id="fingerprint-mismatch"), + pytest.param("orphan_output", False, id="orphan-tool-output"), + pytest.param("unresolved_call", False, id="unresolved-tool-call"), + pytest.param("duplicate_call_id", False, id="duplicate-tool-call-id"), + pytest.param("missing_proof_sentinel", False, id="missing-prefix-proof-fails-closed"), + pytest.param("encrypted_compaction", True, id="encrypted-compaction"), + pytest.param("compaction_missing_durable_proof", False, id="compaction-missing-durable-proof"), + pytest.param("compaction_missing_id", False, id="compaction-missing-id"), + pytest.param("compaction_empty_ciphertext", False, id="compaction-empty-ciphertext"), + pytest.param("compaction_extra_field", False, id="compaction-extra-field"), + pytest.param("compaction_account_scoped_suffix", False, id="compaction-account-scoped-suffix"), + pytest.param("plaintext_summary", False, id="plaintext-summary"), + ], +) +async def test_stream_via_http_bridge_recovers_quarantined_anchor_only_from_safe_full_resend( monkeypatch: pytest.MonkeyPatch, - stored_model: str | None, + request_kind: str, + expected_allowed: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.4", - "instructions": "hi", - "input": "hello", - "prompt_cache_key": "thread-soft", - }, - ) - request_state = proxy_service._WebSocketRequestState( - request_id="req-soft-prompt-cache", - model="gpt-5.4", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=1.0, - event_queue=asyncio.Queue(), - transport="http", - ) + stored_input: list[proxy_service.JsonValue] = [{"role": "user", "content": "first question"}] + tool_call: proxy_service.JsonValue = { + "type": "custom_tool_call", + "call_id": "call_exec", + "name": "exec", + "input": "git status", + "status": "completed", + } + tool_output: proxy_service.JsonValue = { + "type": "custom_tool_call_output", + "call_id": "call_exec", + "output": "clean", + "status": "completed", + } + request_inputs: dict[str, list[proxy_service.JsonValue]] = { + "completed_response": [ + *stored_input, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + }, + {"role": "user", "content": "continue from the full history"}, + ], + "mid_tool_with_user": [ + *stored_input, + { + "type": "message", + "role": "assistant", + "phase": "commentary", + "content": [{"type": "output_text", "text": "Checking the repository."}], + }, + tool_call, + tool_output, + {"type": "message", "role": "developer", "content": "continue safely"}, + {"type": "message", "role": "user", "content": "continue"}, + ], + "mid_tool_without_user": [*stored_input, tool_call, tool_output], + "explicit_conversation": [{"role": "user", "content": "conversation-owned follow-up"}], + "incremental": [{"role": "user", "content": "incremental follow-up only"}], + "fingerprint_mismatch": [ + {"role": "user", "content": "different conversation"}, + tool_call, + tool_output, + ], + "orphan_output": [*stored_input, tool_output], + "unresolved_call": [*stored_input, tool_call], + "duplicate_call_id": [ + *stored_input, + tool_call, + { + "type": "function_call", + "call_id": "call_exec", + "name": "lookup", + "arguments": "{}", + }, + tool_output, + ], + "missing_proof_sentinel": [ + *stored_input, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + }, + {"role": "user", "content": "continue from the full history"}, + ], + "encrypted_compaction": [ + { + "id": "cmp_quarantine_recovery", + "type": "compaction", + "encrypted_content": "encrypted-session-context", + } + ], + "compaction_missing_durable_proof": [ + { + "id": "cmp_without_durable_proof", + "type": "compaction", + "encrypted_content": "encrypted-session-context", + } + ], + "compaction_missing_id": [ + { + "type": "compaction", + "encrypted_content": "encrypted-session-context", + } + ], + "compaction_empty_ciphertext": [ + { + "id": "cmp_empty", + "type": "compaction", + "encrypted_content": "", + } + ], + "compaction_extra_field": [ + { + "id": "cmp_extra", + "type": "compaction", + "encrypted_content": "encrypted-session-context", + "summary": "untrusted", + } + ], + "compaction_account_scoped_suffix": [ + { + "id": "cmp_account_scoped", + "type": "compaction", + "encrypted_content": "encrypted-session-context", + }, + { + "type": "reasoning", + "encrypted_content": "second-owner-bound-ciphertext", + }, + ], + "plaintext_summary": [ + { + "type": "message", + "role": "system", + "content": "Summary of the previous conversation.", + } + ], + } + request_input = request_inputs[request_kind] + payload_data: dict[str, proxy_service.JsonValue] = { + "model": "gpt-5.4", + "instructions": "hi", + "input": request_input, + } + if request_kind == "explicit_conversation": + payload_data["conversation"] = "conv_explicit_continuity" + payload = proxy_service.ResponsesRequest.model_validate(payload_data) + assert isinstance(payload.input, list) + validated_request_input = payload.input + request_state = proxy_service._WebSocketRequestState( + request_id="req-quarantined-full-resend", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", + ) + assert request_state.event_queue is not None + await request_state.event_queue.put(None) + prepared_payloads: list[proxy_service.ResponsesRequest] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del _headers, api_key, api_key_reservation, request_id, client_ip + prepared_payloads.append(prepared_payload) + request_state.previous_response_id = prepared_payload.previous_response_id + return request_state, json.dumps( + { + "type": "response.create", + "model": prepared_payload.model, + "input": prepared_payload.input, + }, + separators=(",", ":"), + ) + + session = _make_bridge_session(key_value="sid-quarantined-full-resend") + session.codex_session = True + session.durable_session_id = "sess-quarantined" + session.durable_owner_epoch = 4 + get_or_create = AsyncMock(return_value=session) + submit = AsyncMock() + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock( + return_value=proxy_service.DurableBridgeLookup( + session_id="sess-quarantined", + canonical_kind="session_header", + canonical_key="sid-quarantined-full-resend", + api_key_scope="__anonymous__", + account_id="acc-bridge", + owner_instance_id=None, + owner_epoch=4, + lease_expires_at=None, + state=HttpBridgeSessionState.CLOSED, + latest_turn_state="http_turn_quarantined", + latest_response_id=( + "resp_without_durable_proof" if request_kind == "compaction_missing_durable_proof" else None + ), + latest_input_item_count=( + None + if request_kind == "compaction_missing_durable_proof" + else (-1 if request_kind == "missing_proof_sentinel" else 1) + ), + latest_input_full_fingerprint=( + None + if request_kind == "compaction_missing_durable_proof" + else ( + "quarantine-sentinel" + if request_kind == "missing_proof_sentinel" + else proxy_service._fingerprint_input_items(stored_input) + ) + ), + latest_pending_tool_calls=None, + ) + ), + ) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + async def collect_chunks() -> list[str]: + return [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-quarantined-full-resend"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + if not expected_allowed: + with pytest.raises(ProxyResponseError) as exc_info: + await collect_chunks() + _assert_http_bridge_full_resend_required(exc_info.value) + if request_kind == "compaction_missing_durable_proof": + assert [prepared.input for prepared in prepared_payloads] == [validated_request_input] + else: + assert prepared_payloads == [] + get_or_create.assert_not_awaited() + submit.assert_not_awaited() + return + + chunks = await collect_chunks() + assert chunks == [] + assert prepared_payloads + assert all(prepared.previous_response_id is None for prepared in prepared_payloads) + assert prepared_payloads[-1].input == validated_request_input + assert request_state.proxy_injected_previous_response_id is False + assert request_state.account_bound_owner_id == ("acc-bridge" if request_kind == "encrypted_compaction" else None) + creation = get_or_create.await_args + assert creation is not None + assert creation.kwargs["previous_response_id"] is None + assert creation.kwargs["preferred_account_id"] == "acc-bridge" + assert creation.kwargs["preferred_account_has_continuity_provenance"] is True + submit_call = submit.await_args + assert submit_call is not None + assert json.loads(submit_call.kwargs["text_data"])["input"] == validated_request_input + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("suffix_kind", "expected_status"), + [ + pytest.param("complete_pair", 200, id="complete-mid-tool-history"), + pytest.param("orphan_output", 400, id="orphan-output-requires-full-resend"), + ], +) +async def test_backend_codex_responses_route_applies_quarantine_tool_history_guard( + monkeypatch: pytest.MonkeyPatch, + suffix_kind: str, + expected_status: int, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + stored_input: list[proxy_service.JsonValue] = [{"role": "user", "content": "first question"}] + tool_call: proxy_service.JsonValue = { + "type": "custom_tool_call", + "call_id": "call_route", + "name": "exec", + "input": "pwd", + "status": "completed", + } + tool_output: proxy_service.JsonValue = { + "type": "custom_tool_call_output", + "call_id": "call_route", + "output": "/workspace", + "status": "completed", + } + request_input = [ + *stored_input, + *([tool_call, tool_output] if suffix_kind == "complete_pair" else [tool_output]), + ] + request_state = proxy_service._WebSocketRequestState( + request_id="req-quarantined-route", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + ) + assert request_state.event_queue is not None + await request_state.event_queue.put( + 'data: {"type":"response.completed","response":{"id":"resp_route_recovered",' + '"status":"completed","output":[]}}\n\n' + ) + await request_state.event_queue.put(None) + prepared_payloads: list[proxy_service.ResponsesRequest] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + **_kwargs: object, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del _headers, api_key, api_key_reservation, request_id, client_ip, _kwargs + prepared_payloads.append(prepared_payload) + request_state.previous_response_id = prepared_payload.previous_response_id + return request_state, json.dumps( + { + "type": "response.create", + "model": prepared_payload.model, + "input": prepared_payload.input, + }, + separators=(",", ":"), + ) + + session = _make_bridge_session(key_value="sid-quarantined-route") + session.codex_session = True + session.durable_session_id = "sess-quarantined-route" + session.durable_owner_epoch = 8 + get_or_create = AsyncMock(return_value=session) + submit = AsyncMock() + record_error = AsyncMock() + record_errors = AsyncMock() + mark_rate_limit = AsyncMock() + mark_quota_exceeded = AsyncMock() + mark_permanent_failure = AsyncMock() + service._load_balancer = cast( + Any, + SimpleNamespace( + record_error=record_error, + record_errors=record_errors, + mark_rate_limit=mark_rate_limit, + mark_quota_exceeded=mark_quota_exceeded, + mark_permanent_failure=mark_permanent_failure, + ), + ) + app_settings = _make_app_settings(sse_keepalive_interval_seconds=0.0) + dashboard_settings = SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast(Any, SimpleNamespace(get=AsyncMock(return_value=dashboard_settings))), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: app_settings) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: app_settings) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock( + return_value=proxy_service.DurableBridgeLookup( + session_id="sess-quarantined-route", + canonical_kind="session_header", + canonical_key="sid-quarantined-route", + api_key_scope="__anonymous__", + account_id="acc-bridge", + owner_instance_id=None, + owner_epoch=8, + lease_expires_at=None, + state=HttpBridgeSessionState.CLOSED, + latest_turn_state="http_turn_quarantined", + latest_response_id=None, + latest_input_item_count=1, + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input), + latest_pending_tool_calls=None, + ) + ), + ) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_register_http_bridge_turn_state", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + monkeypatch.setattr(proxy_api_module, "_select_responses_model_source", AsyncMock(return_value=None)) + monkeypatch.setattr( + proxy_api_module, + "_apply_api_key_enforcement_with_fast_mode_policy", + AsyncMock(return_value=(False, False)), + ) + monkeypatch.setattr(proxy_api_module, "_opportunistic_admission_denial", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_api_module, "_enforce_request_limits", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_api_module, "_rate_limit_headers_for_request", AsyncMock(return_value={})) + monkeypatch.setattr(proxy_api_module, "_release_reservation", AsyncMock()) + + def make_request() -> Request: + return Request( + { + "type": "http", + "method": "POST", + "path": "/backend-api/codex/responses", + "headers": [ + (b"x-codex-session-id", b"sid-quarantined-route"), + (b"user-agent", b"Codex Desktop/0.1.0"), + (b"originator", b"Codex Desktop"), + (b"accept", b"text/event-stream"), + ], + "client": ("203.0.113.9", 54321), + } + ) + + responses = [ + await proxy_api_module.responses( + request=make_request(), + payload={ + "model": "gpt-5.4", + "instructions": "continue", + "input": request_input, + "stream": True, + }, + context=ProxyContext(service=service), + api_key=None, + ) + for _attempt in range(2 if expected_status == 400 else 1) + ] + + assert all(response.status_code == expected_status for response in responses) + if expected_status == 400: + assert [json.loads(bytes(response.body)) for response in responses] == [ + _HTTP_BRIDGE_FULL_RESEND_REQUIRED_PAYLOAD, + _HTTP_BRIDGE_FULL_RESEND_REQUIRED_PAYLOAD, + ] + assert prepared_payloads == [] + get_or_create.assert_not_awaited() + submit.assert_not_awaited() + for health_write in ( + record_error, + record_errors, + mark_rate_limit, + mark_quota_exceeded, + mark_permanent_failure, + ): + health_write.assert_not_awaited() + return + + response = responses[0] + assert isinstance(response, StreamingResponse) + chunks = [chunk async for chunk in response.body_iterator] + assert any("response.completed" in cast(str, chunk) for chunk in chunks) + assert prepared_payloads + assert all(prepared.previous_response_id is None for prepared in prepared_payloads) + get_or_create.assert_awaited_once() + submit.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_model", [None, "gpt-5.3"]) +async def test_stream_via_http_bridge_does_not_prefer_durable_account_for_soft_prompt_cache_lookup( + monkeypatch: pytest.MonkeyPatch, + stored_model: str | None, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": "hello", + "prompt_cache_key": "thread-soft", + }, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-prompt-cache", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + event_queue=asyncio.Queue(), + transport="http", + ) event_queue = request_state.event_queue assert event_queue is not None await event_queue.put(None) @@ -8247,12 +9045,45 @@ def test_http_bridge_drain_detach_removes_old_previous_response_alias() -> None: @pytest.mark.asyncio -async def test_stream_via_http_bridge_does_not_inject_durable_anchor_for_live_turn_state_session( +@pytest.mark.parametrize( + ("request_kind", "has_current_socket_anchor", "expected_allowed"), + [ + pytest.param("incremental", False, False, id="incremental-without-current-socket-anchor"), + pytest.param("verified_full_resend", False, True, id="verified-full-resend"), + pytest.param("anchored_full_resend", True, True, id="current-socket-anchor"), + ], +) +async def test_stream_via_http_bridge_requires_safe_continuity_for_live_turn_state_session( monkeypatch: pytest.MonkeyPatch, + request_kind: str, + has_current_socket_anchor: bool, + expected_allowed: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) + stored_input: list[proxy_service.JsonValue] = [{"role": "user", "content": "first question"}] + request_inputs: dict[str, proxy_service.JsonValue] = { + "incremental": "incremental follow-up", + "verified_full_resend": [ + *stored_input, + { + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "first answer"}], + }, + {"role": "user", "content": "verified follow-up"}, + ], + "anchored_full_resend": [ + *stored_input, + {"role": "user", "content": "current-socket follow-up"}, + ], + } payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": "hello"}, + { + "model": "gpt-5.4", + "instructions": "hi", + "input": request_inputs[request_kind], + }, ) request_state = proxy_service._WebSocketRequestState( request_id="req-live-turn-state", @@ -8267,7 +9098,7 @@ async def test_stream_via_http_bridge_does_not_inject_durable_anchor_for_live_tu event_queue = request_state.event_queue assert event_queue is not None await event_queue.put(None) - captured: dict[str, object] = {} + prepared_payloads: list[proxy_service.ResponsesRequest] = [] def fake_prepare( prepared_payload: proxy_service.ResponsesRequest, @@ -8279,8 +9110,13 @@ def fake_prepare( client_ip: str | None = None, ) -> tuple[proxy_service._WebSocketRequestState, str]: del api_key, api_key_reservation, request_id, client_ip - captured["previous_response_id"] = prepared_payload.previous_response_id - return request_state, '{"type":"response.create"}' + prepared_payloads.append(prepared_payload) + request_state.previous_response_id = prepared_payload.previous_response_id + return request_state, proxy_service._response_create_text( + prepared_payload, + include_type_field=True, + client_metadata=None, + ) session_key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_live", None) session = proxy_service._HTTPBridgeSession( @@ -8300,7 +9136,13 @@ def fake_prepare( queued_request_count=0, last_used_at=1.0, idle_ttl_seconds=120.0, + codex_session=True, ) + if has_current_socket_anchor: + session.last_completed_response_id = "resp_latest" + session.last_completed_response_store = False + session.last_completed_input_count = len(stored_input) + session.last_completed_input_prefix_fingerprint = proxy_service._fingerprint_input_items(stored_input) service._http_bridge_sessions[session_key] = session service._http_bridge_turn_state_index[proxy_service._http_bridge_turn_state_alias_key("http_turn_live", None)] = ( session_key @@ -8340,34 +9182,56 @@ def fake_prepare( state=HttpBridgeSessionState.ACTIVE, latest_turn_state="http_turn_live", latest_response_id="resp_latest", + latest_input_item_count=len(stored_input), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input), ) ), ) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) - monkeypatch.setattr(service, "_submit_http_bridge_request", AsyncMock()) + get_or_create = AsyncMock(return_value=session) + submit = AsyncMock() + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={"x-codex-turn-state": "http_turn_live"}, - codex_session_affinity=True, - propagate_http_errors=False, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ) - ] + async def collect_chunks() -> list[str]: + return [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-turn-state": "http_turn_live"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] - assert chunks == [] - assert captured["previous_response_id"] is None + if not expected_allowed: + with pytest.raises(ProxyResponseError) as exc_info: + await collect_chunks() + _assert_http_bridge_full_resend_required(exc_info.value) + assert len(prepared_payloads) == 1 + get_or_create.assert_awaited_once() + submit.assert_not_awaited() + return + + assert await collect_chunks() == [] + submit.assert_awaited_once() + if has_current_socket_anchor: + assert prepared_payloads[-1].previous_response_id == "resp_latest" + assert prepared_payloads[-1].input == cast(list[proxy_service.JsonValue], payload.input)[len(stored_input) :] + else: + assert all(prepared.previous_response_id is None for prepared in prepared_payloads) + assert prepared_payloads[-1].input == payload.input @pytest.mark.asyncio @@ -8736,13 +9600,40 @@ async def unexpected_forward(**kwargs: object): "retains_prior_output", "takeover_context_matches", "takeover_account_id", + "request_is_incremental", + "takeover_latest_response_id", ), [ - pytest.param(False, True, True, "acc-1", id="local-create-safe-resend"), - pytest.param(True, True, True, "acc-1", id="owner-forward-safe-resend"), - pytest.param(True, False, True, "acc-1", id="owner-forward-unsafe-resend"), - pytest.param(True, False, True, "acc-2", id="owner-forward-refreshed-account"), - pytest.param(True, False, False, "acc-1", id="owner-forward-refreshed-prefix-mismatch"), + pytest.param(False, True, True, "acc-1", False, "resp_latest", id="local-create-safe-resend"), + pytest.param(True, True, True, "acc-1", False, "resp_latest", id="owner-forward-safe-resend"), + pytest.param(True, False, True, "acc-1", False, "resp_latest", id="owner-forward-unsafe-resend"), + pytest.param( + True, + False, + True, + "acc-1", + True, + None, + id="owner-forward-quarantined-incremental", + ), + pytest.param( + True, + False, + True, + "acc-2", + False, + "resp_latest", + id="owner-forward-refreshed-account", + ), + pytest.param( + True, + False, + False, + "acc-1", + False, + "resp_latest", + id="owner-forward-refreshed-prefix-mismatch", + ), ], ) async def test_stream_via_http_bridge_preserves_context_after_owner_unavailable( @@ -8751,25 +9642,31 @@ async def test_stream_via_http_bridge_preserves_context_after_owner_unavailable( retains_prior_output: bool, takeover_context_matches: bool, takeover_account_id: str, + request_is_incremental: bool, + takeover_latest_response_id: str | None, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - prefix_items = [{"role": "user", "content": "one"}] - retained_output = { + prefix_items: list[proxy_service.JsonValue] = [{"role": "user", "content": "one"}] + retained_output: proxy_service.JsonValue = { "type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "two"}], } - input_items = [*prefix_items] + input_items: list[proxy_service.JsonValue] = [*prefix_items] if retains_prior_output: input_items.append(retained_output) input_items.append({"role": "user", "content": "three"}) + payload_input: proxy_service.JsonValue = "incremental follow-up" if request_is_incremental else input_items payload = proxy_service.ResponsesRequest.model_validate( - {"model": "gpt-5.4", "instructions": "hi", "input": input_items}, + {"model": "gpt-5.4", "instructions": "hi", "input": payload_input}, + ) + payload_prefix_items = ( + prefix_items + if request_is_incremental + else cast(list[proxy_service.JsonValue], payload.input)[: len(prefix_items)] ) - payload_prefix_items = cast(list[proxy_service.JsonValue], payload.input)[: len(prefix_items)] request_states: list[proxy_service._WebSocketRequestState] = [] prepared_previous_response_ids: list[str | None] = [] - prepared_inputs: list[proxy_service.JsonValue] = [] def fake_prepare( prepared_payload: proxy_service.ResponsesRequest, @@ -8782,7 +9679,6 @@ def fake_prepare( ) -> tuple[proxy_service._WebSocketRequestState, str]: del api_key, api_key_reservation, request_id, client_ip prepared_previous_response_ids.append(prepared_payload.previous_response_id) - prepared_inputs.append(prepared_payload.input) state = proxy_service._WebSocketRequestState( request_id=f"req-{len(request_states)}", model="gpt-5.4", @@ -8830,99 +9726,398 @@ async def fake_get_or_create_http_bridge_session(*args: object, **kwargs: object key = cast(proxy_service._HTTPBridgeSessionKey, args[0]) return _make_bridge_session(key=key, key_value=key.affinity_key) - forwarded_payloads: list[proxy_service.ResponsesRequest] = [] + forwarded_payloads: list[proxy_service.ResponsesRequest] = [] + + async def fake_forward_http_bridge_request_to_owner(**kwargs: object): + forwarded_payloads.append(cast(proxy_service.ResponsesRequest, kwargs["payload"])) + if False: + yield "" + raise owner_unavailable + + async def fake_stream_http_bridge_session_events(*args: object, **kwargs: object): + del args, kwargs + if False: + yield "" + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: Settings( + http_responses_session_bridge_enabled=True, + http_responses_session_bridge_instance_id="instance-a", + ), + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="sess-fresh-owner-unavailable", + canonical_kind="turn_state_header", + canonical_key="http_turn_fresh", + api_key_scope="__anonymous__", + account_id="acc-1", + owner_instance_id="instance-b", + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_fresh", + latest_response_id="resp_latest", + latest_input_item_count=len(prefix_items), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(payload_prefix_items), + ) + takeover_lookup = replace( + durable_lookup, + account_id=takeover_account_id, + owner_instance_id=None, + lease_expires_at=None, + latest_response_id=takeover_latest_response_id, + latest_input_full_fingerprint=( + durable_lookup.latest_input_full_fingerprint if takeover_context_matches else "f" * 64 + ), + ) + monkeypatch.setattr( + service._durable_bridge, + "lookup_request_targets", + AsyncMock( + side_effect=[durable_lookup, takeover_lookup] + if forward_to_active_owner and not retains_prior_output + else None, + return_value=durable_lookup, + ), + ) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) + monkeypatch.setattr( + service, + "_http_bridge_can_forward_to_active_owner", + AsyncMock(return_value=forward_to_active_owner), + ) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create_http_bridge_session) + monkeypatch.setattr(service, "_forward_http_bridge_request_to_owner", fake_forward_http_bridge_request_to_owner) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_http_bridge_session_events) + + async def collect_chunks() -> list[str]: + return [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={ + "x-codex-session-id": "session-shared-with-retired-owner", + "x-codex-turn-state": "http_turn_fresh", + }, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + if not takeover_context_matches: + with pytest.raises(ProxyResponseError) as exc_info: + await collect_chunks() + assert exc_info.value.payload["error"]["code"] == "bridge_owner_unreachable" + assert get_or_create_calls == 1 + return + if forward_to_active_owner and not retains_prior_output: + with pytest.raises(ProxyResponseError) as exc_info: + await collect_chunks() + _assert_http_bridge_full_resend_required(exc_info.value) + assert get_or_create_calls == 1 + assert forwarded_payloads == [payload] + assert prepared_previous_response_ids == [None] + return + + chunks = await collect_chunks() + + assert chunks == [] + if forward_to_active_owner and retains_prior_output: + assert prepared_previous_response_ids == [None, None, None] + assert forwarded_payloads == [payload] + else: + assert prepared_previous_response_ids == [None, None] + assert forwarded_payloads == [] + assert get_or_create_kwargs[-1]["allow_forward_to_owner"] is False + assert get_or_create_kwargs[-1]["exclude_account_ids"] == ({"acc-1"} if retains_prior_output else None) + assert get_or_create_kwargs[-1]["preferred_account_id"] == (None if retains_prior_output else takeover_account_id) + assert get_or_create_kwargs[-1]["headers"] == ( + {} + if retains_prior_output + else { + "x-codex-session-id": "session-shared-with-retired-owner", + "x-codex-turn-state": "http_turn_fresh", + } + ) + assert request_states[-1].previous_response_id is None + assert request_states[-1].proxy_injected_previous_response_id is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("scenario", "anchor_kind", "request_kind", "expected_error_code"), + [ + pytest.param( + "epoch_advanced", + "bootstrap", + "incremental", + "bridge_owner_unreachable", + id="bootstrap-owner-epoch-advanced", + ), + pytest.param( + "owner_transferred", + "bootstrap", + "incremental", + "bridge_owner_unreachable", + id="bootstrap-owner-transferred", + ), + pytest.param( + "quarantined", + "bootstrap", + "incremental", + "continuity_requires_full_resend", + id="bootstrap-refreshed-quarantine-incremental", + ), + pytest.param( + "quarantined", + "bootstrap", + "mid_tool", + None, + id="bootstrap-refreshed-quarantine-mid-tool", + ), + pytest.param( + "quarantined", + "turn_state", + "mid_tool", + None, + id="turn-state-refreshed-quarantine-mid-tool", + ), + pytest.param( + "quarantined", + "turn_state", + "compaction", + None, + id="turn-state-refreshed-quarantine-compaction", + ), + ], +) +async def test_stream_via_http_bridge_refreshes_durable_state_before_owner_forward_takeover( + monkeypatch: pytest.MonkeyPatch, + scenario: str, + anchor_kind: str, + request_kind: str, + expected_error_code: str | None, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session_header = "sid-owner-forward-refresh" + turn_state = "http_turn_owner_forward_refresh" + headers = {"x-codex-session-id": session_header} + canonical_kind = "session_header" + canonical_key = session_header + if anchor_kind == "turn_state": + headers["x-codex-turn-state"] = turn_state + canonical_kind = "turn_state_header" + canonical_key = turn_state + + stored_input: list[proxy_service.JsonValue] = [{"role": "user", "content": "first question"}] + tool_call: proxy_service.JsonValue = { + "type": "custom_tool_call", + "call_id": "call_exec", + "name": "exec", + "input": "git status", + "status": "completed", + } + tool_output: proxy_service.JsonValue = { + "type": "custom_tool_call_output", + "call_id": "call_exec", + "output": "clean", + "status": "completed", + } + if request_kind == "mid_tool": + request_input: proxy_service.JsonValue = [*stored_input, tool_call, tool_output] + elif request_kind == "compaction": + request_input = [ + { + "id": "cmp_owner_forward_refresh", + "type": "compaction", + "encrypted_content": "encrypted-complete-context", + }, + { + "type": "message", + "role": "user", + "content": "continue after automatic compaction", + }, + ] + else: + request_input = "incremental follow-up" + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": request_input, + } + ) + lease_expires_at = datetime.now(timezone.utc) + timedelta(seconds=60) + initial_lookup = proxy_service.DurableBridgeLookup( + session_id="sess-owner-forward-refresh", + canonical_kind=canonical_kind, + canonical_key=canonical_key, + api_key_scope="__anonymous__", + account_id="acc-original", + owner_instance_id="instance-b", + owner_epoch=4, + lease_expires_at=lease_expires_at, + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state=turn_state, + latest_response_id="resp_original", + latest_input_item_count=len(stored_input), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_input), + model="gpt-5.4", + latest_pending_tool_calls={"call_exec": "custom_tool_call"}, + ) + if scenario == "epoch_advanced": + refreshed_lookup = replace(initial_lookup, owner_epoch=5) + elif scenario == "owner_transferred": + refreshed_lookup = replace( + initial_lookup, + account_id="acc-new", + owner_instance_id="instance-c", + owner_epoch=5, + ) + elif anchor_kind == "turn_state": + refreshed_lookup = replace( + initial_lookup, + owner_instance_id=None, + lease_expires_at=None, + latest_response_id=None, + latest_pending_tool_calls=None, + ) + else: + refreshed_lookup = replace( + initial_lookup, + latest_response_id=None, + latest_pending_tool_calls=None, + ) + + prepared_payloads: list[proxy_service.ResponsesRequest] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del _headers, api_key, api_key_reservation, request_id, client_ip + prepared_payloads.append(prepared_payload) + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-owner-forward-refresh-{len(prepared_payloads)}", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + previous_response_id=prepared_payload.previous_response_id, + transport="http", + ) + return request_state, proxy_service._response_create_text( + prepared_payload, + include_type_field=True, + client_metadata=None, + ) + + owner_forward = proxy_service._HTTPBridgeOwnerForward( + owner_instance="instance-b", + owner_endpoint="http://instance-b", + key=proxy_service._HTTPBridgeSessionKey(canonical_kind, canonical_key, None), + ) + local_session = _make_bridge_session(key=proxy_service._HTTPBridgeSessionKey(canonical_kind, canonical_key, None)) + local_session.account = cast( + Any, + SimpleNamespace(id="acc-original", status=AccountStatus.ACTIVE, plan_type="plus"), + ) + get_or_create = AsyncMock(side_effect=[owner_forward, local_session]) + lookup_request_targets = AsyncMock(side_effect=[initial_lookup, refreshed_lookup]) + streamed_sessions: list[proxy_service._HTTPBridgeSession] = [] - async def fake_forward_http_bridge_request_to_owner(**kwargs: object): - forwarded_payloads.append(cast(proxy_service.ResponsesRequest, kwargs["payload"])) + async def fail_owner_forward(**kwargs: object): + del kwargs if False: yield "" - raise owner_unavailable + raise ProxyResponseError( + 502, + openai_error( + "bridge_owner_unreachable", + "HTTP bridge owner request failed", + error_type="server_error", + ), + ) - async def fake_stream_http_bridge_session_events(*args: object, **kwargs: object): - del args, kwargs + async def stream_local_session( + session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ): + del kwargs + streamed_sessions.append(session) if False: yield "" - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) monkeypatch.setattr( proxy_service, "get_settings", - lambda: Settings( - http_responses_session_bridge_enabled=True, - http_responses_session_bridge_instance_id="instance-a", - ), - ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="sess-fresh-owner-unavailable", - canonical_kind="turn_state_header", - canonical_key="http_turn_fresh", - api_key_scope="__anonymous__", - account_id="acc-1", - owner_instance_id="instance-b", - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_fresh", - latest_response_id="resp_latest", - latest_input_item_count=len(prefix_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(payload_prefix_items), - ) - takeover_lookup = replace( - durable_lookup, - account_id=takeover_account_id, - owner_instance_id=None, - lease_expires_at=None, - latest_input_full_fingerprint=( - durable_lookup.latest_input_full_fingerprint if takeover_context_matches else "f" * 64 - ), + lambda: _make_app_settings(http_responses_session_bridge_instance_id="instance-a"), ) monkeypatch.setattr( - service._durable_bridge, - "lookup_request_targets", - AsyncMock( - side_effect=[durable_lookup, takeover_lookup] - if forward_to_active_owner and not retains_prior_output - else None, - return_value=durable_lookup, + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) ), ) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", lookup_request_targets) monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) - monkeypatch.setattr( - service, - "_http_bridge_can_forward_to_active_owner", - AsyncMock(return_value=forward_to_active_owner), - ) - monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=True)) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create_http_bridge_session) - monkeypatch.setattr(service, "_forward_http_bridge_request_to_owner", fake_forward_http_bridge_request_to_owner) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_http_bridge_session_events) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_forward_http_bridge_request_to_owner", fail_owner_forward) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", stream_local_session) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) async def collect_chunks() -> list[str]: return [ chunk async for chunk in service._stream_via_http_bridge( payload, - headers={ - "x-codex-session-id": "session-shared-with-retired-owner", - "x-codex-turn-state": "http_turn_fresh", - }, + headers=headers, codex_session_affinity=True, propagate_http_errors=False, openai_cache_affinity=False, @@ -8936,47 +10131,32 @@ async def collect_chunks() -> list[str]: ) ] - if not takeover_context_matches: + if expected_error_code is not None: with pytest.raises(ProxyResponseError) as exc_info: await collect_chunks() - assert exc_info.value.payload["error"]["code"] == "bridge_owner_unreachable" - assert get_or_create_calls == 1 + if expected_error_code == "continuity_requires_full_resend": + _assert_http_bridge_full_resend_required(exc_info.value) + else: + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == expected_error_code + assert get_or_create.await_count == 1 + assert streamed_sessions == [] + assert all(prepared.previous_response_id is None for prepared in prepared_payloads) + assert lookup_request_targets.await_count == 2 return - chunks = await collect_chunks() - - assert chunks == [] - if forward_to_active_owner and retains_prior_output: - assert prepared_previous_response_ids == [None, None, None] - assert forwarded_payloads == [payload] - elif forward_to_active_owner: - assert prepared_previous_response_ids == [None, "resp_latest"] - assert forwarded_payloads == [payload] - normalized_input = cast(list[proxy_service.JsonValue], payload.input) - assert prepared_inputs[-1] == normalized_input[len(prefix_items) :] - else: - assert prepared_previous_response_ids == [None, None] - assert forwarded_payloads == [] - assert get_or_create_kwargs[-1]["allow_forward_to_owner"] is False - assert get_or_create_kwargs[-1]["exclude_account_ids"] == ({"acc-1"} if retains_prior_output else None) - assert get_or_create_kwargs[-1]["preferred_account_id"] == (None if retains_prior_output else takeover_account_id) - assert get_or_create_kwargs[-1]["headers"] == ( - {} - if retains_prior_output - else { - "x-codex-session-id": "session-shared-with-retired-owner", - "x-codex-turn-state": "http_turn_fresh", - } - ) - assert request_states[-1].previous_response_id == ( - "resp_latest" if forward_to_active_owner and not retains_prior_output else None - ) - assert request_states[-1].proxy_injected_previous_response_id is ( - forward_to_active_owner and not retains_prior_output - ) - if forward_to_active_owner and not retains_prior_output: - assert request_states[-1].fresh_upstream_request_is_retry_safe is False - assert request_states[-1].input_item_count == len(input_items) + assert await collect_chunks() == [] + assert lookup_request_targets.await_count == 2 + assert get_or_create.await_count == 2 + recovery_call = get_or_create.await_args_list[-1] + assert recovery_call.kwargs["allow_forward_to_owner"] is False + assert recovery_call.kwargs["allow_bootstrap_owner_rebind"] is True + assert recovery_call.kwargs["durable_lookup"] == refreshed_lookup + assert recovery_call.kwargs["preferred_account_id"] == "acc-original" + assert recovery_call.kwargs["preferred_account_has_continuity_provenance"] is True + assert recovery_call.kwargs["previous_response_id"] is None + assert streamed_sessions == [local_session] + assert all(prepared.previous_response_id is None for prepared in prepared_payloads) @pytest.mark.asyncio @@ -9244,6 +10424,7 @@ async def fake_get_or_create_http_bridge_session(*args: object, **kwargs: object codex_idle_ttl_seconds=1800.0, max_sessions=8, queue_limit=4, + forwarded_proxy_injected_previous_response_id=True, ) ] @@ -9255,6 +10436,7 @@ async def fake_get_or_create_http_bridge_session(*args: object, **kwargs: object surface="http_bridge", ) assert captured_preferred["value"] == "acc-owner-from-logs" + assert request_state.proxy_injected_previous_response_id is True @pytest.mark.asyncio @@ -9510,9 +10692,9 @@ async def test_http_bridge_waits_for_registration_for_hard_keys_before_startup_c @pytest.mark.parametrize( - ("request_headers", "expected_turn_state", "expected_unanchored"), + ("request_headers", "expected_turn_state", "expected_unanchored", "expected_proxy_injected"), [ - ({"x-codex-session-id": "sid-123"}, "http_turn_generated", True), + ({"x-codex-session-id": "sid-123"}, "http_turn_generated", True, False), ( { "x-codex-session-id": "sid-123", @@ -9520,6 +10702,7 @@ async def test_http_bridge_waits_for_registration_for_hard_keys_before_startup_c }, "http_turn_generated", True, + False, ), ( { @@ -9528,6 +10711,16 @@ async def test_http_bridge_waits_for_registration_for_hard_keys_before_startup_c }, "http_turn_generated", True, + False, + ), + ( + { + "x-codex-session-id": "sid-123", + "x-codex-turn-state": "http_turn_client", + }, + "http_turn_client", + False, + False, ), ( { @@ -9536,6 +10729,7 @@ async def test_http_bridge_waits_for_registration_for_hard_keys_before_startup_c }, "http_turn_client", False, + True, ), ], ) @@ -9545,6 +10739,7 @@ async def test_forward_http_bridge_request_to_owner_preserves_session_header_key request_headers: dict[str, str], expected_turn_state: str, expected_unanchored: bool, + expected_proxy_injected: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) owner_forward = proxy_service._HTTPBridgeOwnerForward( @@ -9552,7 +10747,14 @@ async def test_forward_http_bridge_request_to_owner_preserves_session_header_key owner_endpoint="http://instance-b", key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-123", None), ) - payload = proxy_service.ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": "hi"}) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": "hi", + **({"previous_response_id": "resp_proxy_injected"} if expected_proxy_injected else {}), + } + ) captured: dict[str, object] = {} async def fake_stream_responses(**kwargs: object): @@ -9579,6 +10781,7 @@ async def fake_stream_responses(**kwargs: object): downstream_turn_state="http_turn_generated", request_started_at=10.0, proxy_api_authorization=None, + proxy_injected_previous_response_id=expected_proxy_injected, ) ] @@ -9586,6 +10789,7 @@ async def fake_stream_responses(**kwargs: object): context = cast(proxy_service.HTTPBridgeForwardContext, captured["context"]) assert context.downstream_turn_state == expected_turn_state assert context.original_request_unanchored is expected_unanchored + assert context.proxy_injected_previous_response_id is expected_proxy_injected assert context.original_affinity_kind == "session_header" assert context.original_affinity_key == "sid-123" assert cast(dict[str, str], captured["headers"])["x-codex-session-id"] == "sid-123" @@ -10082,8 +11286,14 @@ async def fake_forward(**kwargs: object): @pytest.mark.asyncio +@pytest.mark.parametrize( + "proxy_injected_anchor", + [False, True], + ids=["client-supplied-anchor", "proxy-injected-anchor"], +) async def test_stream_via_http_bridge_reacquires_api_key_reservation_for_local_previous_response_rebind( monkeypatch: pytest.MonkeyPatch, + proxy_injected_anchor: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) started_at = time.monotonic() @@ -10121,6 +11331,7 @@ async def test_stream_via_http_bridge_reacquires_api_key_reservation_for_local_p ) request_state_initial.request_stage = "follow_up" request_state_initial.preferred_account_id = "acc-1" + request_state_initial.proxy_injected_previous_response_id = proxy_injected_anchor request_state_retry = proxy_service._WebSocketRequestState( request_id="req-retry", model="gpt-5.4", @@ -10264,6 +11475,7 @@ async def fake_stream_http_bridge_session_events( assert chunks[-1] == 'data: {"type":"response.completed"}\n\n' assert get_or_create.await_count == 3 assert prepare_reservations == [initial_reservation, retried_reservation] + assert request_state_retry.proxy_injected_previous_response_id is proxy_injected_anchor reserve_retry.assert_awaited_once() @@ -10497,8 +11709,20 @@ async def test_http_bridge_local_owner_rejects_aliases_for_distinct_live_session @pytest.mark.asyncio +@pytest.mark.parametrize( + ("proxy_injected_anchor", "retry_previous_response_id", "expected_retry_provenance"), + [ + pytest.param(False, "resp_prev_1", False, id="client-supplied-anchor"), + pytest.param(True, "resp_prev_1", True, id="same-proxy-injected-anchor"), + pytest.param(True, "resp_changed", False, id="changed-anchor"), + pytest.param(True, None, False, id="removed-anchor"), + ], +) async def test_stream_via_http_bridge_reacquires_api_key_reservation_after_owner_forward_failure( monkeypatch: pytest.MonkeyPatch, + proxy_injected_anchor: bool, + retry_previous_response_id: str | None, + expected_retry_provenance: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) started_at = time.monotonic() @@ -10531,10 +11755,13 @@ async def test_stream_via_http_bridge_reacquires_api_key_reservation_after_owner started_at=started_at, event_queue=asyncio.Queue(), transport="http", - previous_response_id="resp_prev_1", + previous_response_id=retry_previous_response_id, + fresh_upstream_request_text='{"type":"response.create","request":"safe-full-history"}', + fresh_upstream_request_is_retry_safe=True, ) request_state_initial.request_stage = "follow_up" request_state_initial.preferred_account_id = "acc-1" + request_state_initial.proxy_injected_previous_response_id = proxy_injected_anchor request_state_retry = proxy_service._WebSocketRequestState( request_id="req-retry", model="gpt-5.4", @@ -10680,6 +11907,11 @@ async def produce_after_reattach_delay() -> None: assert get_or_create.await_count == 3 assert prepare_reservations == [initial_reservation, retried_reservation] assert submitted_reservations == [retried_reservation] + assert request_state_retry.proxy_injected_previous_response_id is expected_retry_provenance + assert request_state_retry.fresh_upstream_request_text == ( + request_state_initial.fresh_upstream_request_text if expected_retry_provenance else None + ) + assert request_state_retry.fresh_upstream_request_is_retry_safe is expected_retry_provenance reserve_retry.assert_awaited_once() @@ -10943,29 +12175,168 @@ def fake_prepare( transport="http", previous_response_id="resp_prev_1", ) - return state, '{"type":"response.create"}' + return state, '{"type":"response.create"}' + + submit_calls = 0 + + async def fake_submit_http_bridge_request( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + nonlocal submit_calls + del _session, text_data, queue_limit + submit_calls += 1 + if submit_calls == 1: + raise ProxyResponseError(400, proxy_service.openai_error("previous_response_not_found", "missing")) + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put('data: {"type":"response.completed"}\n\n') + await event_queue.put(None) + + get_or_create = AsyncMock(side_effect=[failing_session, retry_session]) + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + monkeypatch.setattr(service, "_reset_http_bridge_session_after_local_terminal_error", AsyncMock()) + monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"x-codex-session-id": "sid-recover"}, + codex_session_affinity=True, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=900.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert get_or_create.await_count == 2 + assert submit_calls == 2 + synthetic_item = { + "type": "custom_tool_call_output", + "call_id": "call_custom_shell", + "output": ( + "Tool call was not executed because the previous turn was interrupted before tool output was available." + ), + } + assert len(prepared_inputs) == 3 + assert prepared_inputs[0] == input_items + assert prepared_inputs[1] == [synthetic_item, *input_items] + assert prepared_inputs[2] == [synthetic_item, *input_items] + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_local_recovery_preserves_safe_full_history_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + started_at = time.monotonic() + stored_input: list[proxy_service.JsonValue] = [{"role": "user", "content": "first"}] + full_input: list[proxy_service.JsonValue] = [ + *stored_input, + {"role": "assistant", "content": "answer"}, + {"role": "user", "content": "follow up"}, + ] + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "input": full_input, + "store": False, + } + ) + failing_session = _make_owner_forward_recovery_session() + failing_session.codex_session = True + failing_session.last_completed_response_id = "resp_prev_1" + failing_session.last_completed_response_store = False + failing_session.last_completed_input_count = len(stored_input) + failing_session.last_completed_input_prefix_fingerprint = proxy_service._fingerprint_input_items(stored_input) + retry_session = _make_owner_forward_recovery_session() + + prepared_payloads: list[proxy_service.ResponsesRequest] = [] + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del _headers, api_key, api_key_reservation, request_id, client_ip + prepared_payloads.append(prepared_payload) + text_data = proxy_service._response_create_text( + prepared_payload, + include_type_field=True, + client_metadata=None, + ) + return ( + proxy_service._WebSocketRequestState( + request_id=f"req-{len(prepared_payloads)}", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=started_at, + event_queue=asyncio.Queue(), + request_text=text_data, + previous_response_id=prepared_payload.previous_response_id, + response_store=prepared_payload.store, + transport="http", + ), + text_data, + ) - submit_calls = 0 + streamed_states: list[proxy_service._WebSocketRequestState] = [] - async def fake_submit_http_bridge_request( + async def fake_stream_http_bridge_session_events( _session: proxy_service._HTTPBridgeSession, *, request_state: proxy_service._WebSocketRequestState, - text_data: str, - queue_limit: int, - ) -> None: - nonlocal submit_calls - del _session, text_data, queue_limit - submit_calls += 1 - if submit_calls == 1: + **kwargs: object, + ): + del kwargs + streamed_states.append(request_state) + if len(streamed_states) == 1: raise ProxyResponseError(400, proxy_service.openai_error("previous_response_not_found", "missing")) - event_queue = request_state.event_queue - assert event_queue is not None - await event_queue.put('data: {"type":"response.completed"}\n\n') - await event_queue.put(None) + if False: + yield "" get_or_create = AsyncMock(side_effect=[failing_session, retry_session]) - monkeypatch.setattr( proxy_service, "get_settings_cache", @@ -10985,10 +12356,9 @@ async def fake_submit_http_bridge_request( ) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-1")) monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) - monkeypatch.setattr(service, "_submit_http_bridge_request", fake_submit_http_bridge_request) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_http_bridge_session_events) monkeypatch.setattr(service, "_reset_http_bridge_session_after_local_terminal_error", AsyncMock()) monkeypatch.setattr(service, "_detach_http_bridge_request", AsyncMock()) @@ -11010,20 +12380,19 @@ async def fake_submit_http_bridge_request( ) ] - assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert chunks == [] assert get_or_create.await_count == 2 - assert submit_calls == 2 - synthetic_item = { - "type": "custom_tool_call_output", - "call_id": "call_custom_shell", - "output": ( - "Tool call was not executed because the previous turn was interrupted before tool output was available." - ), - } - assert len(prepared_inputs) == 3 - assert prepared_inputs[0] == input_items - assert prepared_inputs[1] == [synthetic_item, *input_items] - assert prepared_inputs[2] == [synthetic_item, *input_items] + assert len(streamed_states) == 2 + initial_state, retry_state = streamed_states + assert initial_state.proxy_injected_previous_response_id is True + assert initial_state.fresh_upstream_request_is_retry_safe is True + assert initial_state.fresh_upstream_request_text is not None + fallback_payload = json.loads(initial_state.fresh_upstream_request_text) + assert "previous_response_id" not in fallback_payload + assert fallback_payload["input"] == payload.to_payload()["input"] + assert retry_state.proxy_injected_previous_response_id is True + assert retry_state.fresh_upstream_request_is_retry_safe is True + assert retry_state.fresh_upstream_request_text == initial_state.fresh_upstream_request_text @pytest.mark.asyncio @@ -12948,6 +14317,45 @@ async def test_get_or_create_http_bridge_session_recovers_locally_when_owner_end service._ring_membership.resolve_endpoint.assert_awaited_once_with("instance-b") +@pytest.mark.asyncio +async def test_get_or_create_http_bridge_session_rejects_fresh_creation_when_owner_endpoint_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_123", None) + create_http_bridge_session = AsyncMock() + claim_durable = AsyncMock() + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(service, "_create_http_bridge_session", create_http_bridge_session) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", claim_durable) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-b")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), + ) + service._ring_membership = cast(Any, SimpleNamespace(resolve_endpoint=AsyncMock(return_value=None))) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-turn-state": "http_turn_123"}, + affinity=proxy_service._AffinityPolicy(key="http_turn_123"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + allow_forward_to_owner=True, + allow_fresh_session_creation=False, + ) + + _assert_http_bridge_full_resend_required(exc_info.value) + create_http_bridge_session.assert_not_awaited() + claim_durable.assert_not_awaited() + service._ring_membership.resolve_endpoint.assert_awaited_once_with("instance-b") + + @pytest.mark.asyncio async def test_get_or_create_http_bridge_session_recovers_locally_when_stale_owner_endpoint_is_current( monkeypatch: pytest.MonkeyPatch, @@ -16164,66 +17572,347 @@ async def test_submit_http_bridge_request_does_not_send_after_retirement_between service_tier=None, reasoning_effort=None, api_key_reservation=None, - started_at=1.0, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', + transport="http", + skip_request_log=True, + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_retire_gap", None), + headers={"x-codex-turn-state": "http_turn_retire_gap"}, + affinity=proxy_service._AffinityPolicy( + key="http_turn_retire_gap", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.5", + account=cast(Any, SimpleNamespace(id="acc-retire-gap", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + service._http_bridge_sessions[session.key] = session + stale_send_seen = False + + async def send_text(_text: str) -> None: + nonlocal stale_send_seen + stale_send_seen = service._http_bridge_sessions.get(session.key) is not session or session.closed + + cast(Any, session.upstream).send_text.side_effect = send_text + + class RetireAfterValidationLock: + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, *_exc: object) -> None: + if request_state.response_create_gate_acquired and request_state not in session.pending_requests: + session.closed = True + service._http_bridge_sessions.pop(session.key, None) + return None + + service._http_bridge_lock = cast(Any, RetireAfterValidationLock()) + + try: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + except proxy_service.ProxyResponseError: + pass + finally: + if request_state.response_create_gate_acquired: + await proxy_service._release_websocket_response_create_gate(request_state, session.response_create_gate) + + assert stale_send_seen is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fresh_request_is_retry_safe", + [ + pytest.param(False, id="incremental-fails-closed"), + pytest.param(True, id="full-history-drops-stale-anchor"), + ], +) +async def test_submit_http_bridge_request_revalidates_proxy_anchor_after_gate_wait( + fresh_request_is_retry_safe: bool, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + anchored_text = json.dumps( + { + "type": "response.create", + "previous_response_id": "resp-current-socket", + "input": [{"role": "user", "content": "follow up"}], + }, + separators=(",", ":"), + ) + fresh_text = json.dumps( + { + "type": "response.create", + "input": [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "answer"}, + {"role": "user", "content": "follow up"}, + ], + }, + separators=(",", ":"), + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-anchor-waiter", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text=anchored_text, + previous_response_id="resp-current-socket", + response_store=False, + proxy_injected_previous_response_id=True, + fresh_upstream_request_text=fresh_text, + fresh_upstream_request_is_retry_safe=fresh_request_is_retry_safe, + transport="http", + skip_request_log=True, + ) + gate = asyncio.Semaphore(1) + await gate.acquire() + old_send_text = AsyncMock() + replacement_send_text = AsyncMock() + session = _make_bridge_session( + key_value="bridge-anchor-waiter", + ) + session.response_create_gate = gate + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=old_send_text, close=AsyncMock()), + ) + session.last_completed_response_id = "resp-current-socket" + session.last_completed_response_store = False + service._http_bridge_sessions[session.key] = session + + submit_task = asyncio.create_task( + service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=anchored_text, + queue_limit=8, + ) + ) + try: + with anyio.fail_after(1.0): + while session.admission_waiter_count != 1: + await asyncio.sleep(0) + + async with session.lifecycle_lock: + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=replacement_send_text, close=AsyncMock()), + ) + session.last_completed_response_id = None + session.last_completed_response_store = None + gate.release() + + if fresh_request_is_retry_safe: + await submit_task + replacement_send_text.assert_awaited_once_with(fresh_text) + assert request_state.previous_response_id is None + assert request_state.proxy_injected_previous_response_id is False + assert list(session.pending_requests) == [request_state] + await service._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=True, + request_enqueued=True, + counted_in_queue=True, + ) + else: + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await submit_task + _assert_http_bridge_full_resend_required(exc_info.value) + replacement_send_text.assert_not_awaited() + assert list(session.pending_requests) == [] + assert session.queued_request_count == 0 + assert request_state.response_create_gate is None + assert request_state.response_create_gate_acquired is False + finally: + if not submit_task.done(): + submit_task.cancel() + with pytest.raises(asyncio.CancelledError): + await submit_task + if gate.locked(): + gate.release() + + old_send_text.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_completion_provenance_waits_for_atomic_anchor_validation_and_send( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _BlockingSendUpstream: + def __init__(self) -> None: + self.send_started = asyncio.Event() + self.release_send = asyncio.Event() + self.sent_texts: list[str] = [] + + async def send_text(self, text: str) -> None: + self.send_started.set() + await self.release_send.wait() + self.sent_texts.append(text) + + async def close(self, code: int = 1000, reason: str = "") -> None: + del code, reason + self.release_send.set() + + service = proxy_service.ProxyService(cast(Any, nullcontext())) + completion_registration_finished = asyncio.Event() + + async def register_previous_response_id( + target_session: proxy_service._HTTPBridgeSession, + response_id: str, + *, + input_item_count: int | None = None, + input_full_fingerprint: str | None = None, + pending_tool_calls: dict[str, str] | None = None, + ) -> bool: + assert response_id == "resp-completing" + assert input_item_count == 3 + assert input_full_fingerprint == "fingerprint-completing" + assert pending_tool_calls == {"call-completing": "function_call"} + assert target_session is session + completion_registration_finished.set() + return True + + monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous_response_id) + monkeypatch.setattr(service, "_finalize_websocket_request_state", AsyncMock()) + + completing_request = proxy_service._WebSocketRequestState( + request_id="req-completing", + response_id="resp-completing", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + response_store=False, + input_item_count=3, + input_full_fingerprint="fingerprint-completing", + pending_tool_call_types={"call-completing": "function_call"}, + added_tool_call_types={"call-completing": "function_call"}, + transport="http", + skip_request_log=True, + ) + anchored_text = json.dumps( + { + "type": "response.create", + "previous_response_id": "resp-anchor-before-send", + "input": [{"role": "user", "content": "follow up"}], + }, + separators=(",", ":"), + ) + anchored_request = proxy_service._WebSocketRequestState( + request_id="req-anchor-concurrent-completion", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), awaiting_response_created=True, event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', + request_text=anchored_text, + previous_response_id="resp-anchor-before-send", + response_store=False, + proxy_injected_previous_response_id=True, transport="http", skip_request_log=True, ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_retire_gap", None), - headers={"x-codex-turn-state": "http_turn_retire_gap"}, - affinity=proxy_service._AffinityPolicy( - key="http_turn_retire_gap", - kind=proxy_service.StickySessionKind.CODEX_SESSION, - ), - request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-retire-gap", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, + upstream = _BlockingSendUpstream() + session = _make_bridge_session( + key_value="bridge-anchor-concurrent-completion", + pending_requests=deque([completing_request]), + queued_request_count=1, ) + session.upstream = cast(UpstreamWebSocket, upstream) + session.last_completed_response_id = "resp-anchor-before-send" + session.last_completed_response_store = False + session.last_completed_input_count = 1 + session.last_completed_input_prefix_fingerprint = "fingerprint-before-send" + session.last_pending_tool_calls = {"call-before-send": "function_call"} service._http_bridge_sessions[session.key] = session - stale_send_seen = False - - async def send_text(_text: str) -> None: - nonlocal stale_send_seen - stale_send_seen = service._http_bridge_sessions.get(session.key) is not session or session.closed - cast(Any, session.upstream).send_text.side_effect = send_text - - class RetireAfterValidationLock: - async def __aenter__(self) -> None: - return None - - async def __aexit__(self, *_exc: object) -> None: - if request_state.response_create_gate_acquired and request_state not in session.pending_requests: - session.closed = True - service._http_bridge_sessions.pop(session.key, None) - return None - - service._http_bridge_lock = cast(Any, RetireAfterValidationLock()) - - try: - await service._submit_http_bridge_request( + submit_task = asyncio.create_task( + service._submit_http_bridge_request( session, - request_state=request_state, - text_data=request_state.request_text or "{}", + request_state=anchored_request, + text_data=anchored_text, queue_limit=8, ) - except proxy_service.ProxyResponseError: - pass + ) + completion_task: asyncio.Task[None] | None = None + try: + await asyncio.wait_for(upstream.send_started.wait(), timeout=1.0) + completion_task = asyncio.create_task( + service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp-completing", + "object": "response", + "status": "completed", + "output": [ + { + "type": "function_call", + "call_id": "call-completing", + "name": "lookup", + "arguments": "{}", + } + ], + }, + }, + separators=(",", ":"), + ), + ) + ) + await asyncio.wait_for(completion_registration_finished.wait(), timeout=1.0) + await asyncio.sleep(0) + + assert completion_task.done() is False + assert session.last_completed_response_id == "resp-anchor-before-send" + assert session.last_completed_response_store is False + assert session.last_completed_input_count == 1 + assert session.last_completed_input_prefix_fingerprint == "fingerprint-before-send" + assert session.last_pending_tool_calls == {"call-before-send": "function_call"} finally: - if request_state.response_create_gate_acquired: - await proxy_service._release_websocket_response_create_gate(request_state, session.response_create_gate) + upstream.release_send.set() + await asyncio.wait_for(submit_task, timeout=1.0) + if completion_task is not None: + await asyncio.wait_for(completion_task, timeout=1.0) + + assert upstream.sent_texts == [anchored_text] + assert session.last_completed_response_id == "resp-completing" + assert session.last_completed_response_store is False + assert session.last_completed_input_count == 3 + assert session.last_completed_input_prefix_fingerprint == "fingerprint-completing" + assert session.last_pending_tool_calls == {"call-completing": "function_call"} - assert stale_send_seen is False + await service._cleanup_http_bridge_submit_interruption( + session, + request_state=anchored_request, + gate_acquired=True, + request_enqueued=True, + counted_in_queue=True, + ) @pytest.mark.asyncio @@ -17796,9 +19485,13 @@ async def fake_stream_events( if unsafe_replay_input == "missing_owner": assert exc_info.value.status_code == 502 assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + elif unsafe_replay_input in {"missing_prior_output", "orphan_output"}: + _assert_http_bridge_full_resend_required(exc_info.value) else: assert exc_info.value is owner_unavailable - assert get_or_create.await_count == (0 if unsafe_replay_input == "missing_owner" else 1) + assert get_or_create.await_count == ( + 0 if unsafe_replay_input in {"missing_owner", "missing_prior_output", "orphan_output"} else 1 + ) if unsafe_replay_input == "conversation": last_call = get_or_create.await_args assert last_call is not None @@ -18204,109 +19897,427 @@ async def test_get_or_create_http_bridge_session_prevents_forward_loops( AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), ) - with pytest.raises(ProxyResponseError) as exc_info: - await service._get_or_create_http_bridge_session( - key, - headers={"x-codex-turn-state": "http_turn_123"}, - affinity=proxy_service._AffinityPolicy(key="http_turn_123"), - api_key=None, - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - allow_forward_to_owner=True, - forwarded_request=True, - ) + with pytest.raises(ProxyResponseError) as exc_info: + await service._get_or_create_http_bridge_session( + key, + headers={"x-codex-turn-state": "http_turn_123"}, + affinity=proxy_service._AffinityPolicy(key="http_turn_123"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + allow_forward_to_owner=True, + forwarded_request=True, + ) + + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "bridge_forward_loop_prevented" + create_http_bridge_session.assert_not_awaited() + claim_durable.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_get_or_create_http_bridge_session_replaces_live_session_when_scope_becomes_empty( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + key = proxy_service._HTTPBridgeSessionKey("request", "bridge-key", "key-1") + stale_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4-mini", + account=cast(Any, SimpleNamespace(id="acc-stale", status=AccountStatus.ACTIVE, plan_type="plus")), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + replacement_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=2.0, + idle_ttl_seconds=120.0, + ) + service._http_bridge_sessions[key] = stale_session + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr( + service, + "_create_http_bridge_session", + AsyncMock(return_value=replacement_session), + ) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings(), + ) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a"])), + ) + close_session = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session", close_session) + + reused = await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-key"), + api_key=_make_api_key( + key_id="key-1", + assigned_account_ids=[], + account_assignment_scope_enabled=True, + ), + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + + assert reused is replacement_session + assert service._http_bridge_sessions[key] is replacement_session + assert stale_session.closed is True + await _wait_for_close_await(close_session, stale_session) + + +def test_http_bridge_disconnect_quarantine_selector_uses_only_sent_unambiguous_store_false_anchor() -> None: + def make_request( + request_id: str, + previous_response_id: str, + *, + sent: bool = True, + response_store: bool | None = False, + proxy_injected: bool = True, + ) -> proxy_service._WebSocketRequestState: + return proxy_service._WebSocketRequestState( + request_id=request_id, + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + previous_response_id=previous_response_id, + response_store=response_store, + proxy_injected_previous_response_id=proxy_injected, + response_create_sent_at=time.monotonic() if sent else None, + ) + + sent = make_request("req-sent", "resp-sent") + queued = make_request("req-queued", "resp-sent", sent=False) + persisted = make_request("req-persisted", "resp-sent", response_store=True) + unknown_store = make_request("req-unknown-store", "resp-sent", response_store=None) + client_supplied = make_request("req-client", "resp-sent", proxy_injected=False) + + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [sent, queued, persisted, unknown_store, client_supplied], + latest_response_id=None, + latest_response_store=None, + ) + == "resp-sent" + ) + second_anchor = make_request("req-second", "resp-second") + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [sent, second_anchor], + latest_response_id=None, + latest_response_store=None, + ) + is None + ) + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [sent, second_anchor], + latest_response_id="resp-second", + latest_response_store=None, + ) + == "resp-second" + ) + sent.response_create_gate_acquired = True + sent.awaiting_response_created = True + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [sent, second_anchor], + latest_response_id="resp-second", + latest_response_store=None, + ) + == "resp-sent" + ) + same_anchor = make_request("req-same-anchor", "resp-sent") + sent.response_create_gate_acquired = False + sent.awaiting_response_created = False + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [sent, same_anchor], + latest_response_id=None, + latest_response_store=None, + ) + == "resp-sent" + ) + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [], + latest_response_id="resp-idle", + latest_response_store=False, + ) + == "resp-idle" + ) + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [], + latest_response_id="resp-loaded-from-prior-socket", + latest_response_store=None, + ) + is None + ) + assert ( + http_bridge_helpers_module._select_http_bridge_disconnect_anchor_quarantine_response_id( + [], + latest_response_id="resp-persisted", + latest_response_store=True, + ) + is None + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "close_code", + [None, 1000, 1001, 1012], + ids=["no-close-frame", "normal-close", "going-away", "service-restart"], +) +@pytest.mark.parametrize( + ("persisted_latest_response_id", "expected_local_response_id"), + [ + pytest.param(None, None, id="cas-cleared"), + pytest.param("resp-newer", "resp-connection-local", id="newer-anchor-preserved"), + ], +) +async def test_http_bridge_disconnect_quarantines_sent_store_false_anchor_before_settlement( + monkeypatch: pytest.MonkeyPatch, + close_code: int | None, + persisted_latest_response_id: str | None, + expected_local_response_id: str | None, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-disconnect-quarantine", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + previous_response_id="resp-connection-local", + response_store=False, + proxy_injected_previous_response_id=True, + response_create_sent_at=time.monotonic(), + ) + upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock( + return_value=UpstreamWebSocketMessage( + kind="close", + close_code=close_code, + error="no close frame" if close_code is None else None, + ) + ), + close=AsyncMock(), + ), + ) + session = _make_bridge_session( + key_value=f"disconnect-quarantine-{close_code}", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = upstream + session.durable_session_id = "durable-disconnect-quarantine" + session.durable_owner_epoch = 9 + session.last_completed_response_id = "resp-connection-local" + session.last_completed_response_store = False + session.last_pending_tool_calls = {"call-old": "custom_tool_call"} + settings = _make_app_settings(http_responses_session_bridge_instance_id="instance-disconnect-quarantine") + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + call_order: list[str] = [] + + async def retry_precreated_request(_session: proxy_service._HTTPBridgeSession) -> bool: + call_order.append("retry") + return False + + retry_precreated = AsyncMock(side_effect=retry_precreated_request) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + + async def clear_latest_anchor(**kwargs: object) -> object: + del kwargs + call_order.append("quarantine") + return SimpleNamespace( + owner_instance_id="instance-disconnect-quarantine", + owner_epoch=9, + latest_response_id=persisted_latest_response_id, + ) + + clear = AsyncMock(side_effect=clear_latest_anchor) + monkeypatch.setattr( + service._durable_bridge, + "clear_latest_response_anchor_if_current", + clear, + raising=False, + ) + + async def fail_reader( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> bool: + del kwargs + call_order.append("settle") + target_session.closed = True + return True - assert exc_info.value.status_code == 503 - assert exc_info.value.payload["error"]["code"] == "bridge_forward_loop_prevented" - create_http_bridge_session.assert_not_awaited() - claim_durable.assert_not_awaited() + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + await service._relay_http_bridge_upstream_messages(session) + + retry_precreated.assert_awaited_once_with(session) + clear.assert_awaited_once_with( + session_id="durable-disconnect-quarantine", + instance_id="instance-disconnect-quarantine", + owner_epoch=9, + expected_response_id="resp-connection-local", + ) + assert call_order == ["quarantine", "retry", "settle"] + assert session.last_completed_response_id == expected_local_response_id + expected_local_response_store = None if expected_local_response_id is None else False + assert session.last_completed_response_store is expected_local_response_store + assert session.last_pending_tool_calls == ( + {} if expected_local_response_id is None else {"call-old": "custom_tool_call"} + ) + assert request_state.replay_count == 0 + assert request_state.preferred_account_id is None + assert request_state.excluded_account_ids == set() @pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_replaces_live_session_when_scope_becomes_empty( +async def test_http_bridge_idle_disconnect_quarantines_current_socket_store_false_response( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("request", "bridge-key", "key-1") - stale_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4-mini", - account=cast(Any, SimpleNamespace(id="acc-stale", status=AccountStatus.ACTIVE, plan_type="plus")), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, + upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock( + return_value=UpstreamWebSocketMessage( + kind="close", + close_code=1000, + ) + ), + close=AsyncMock(), + ), ) - replacement_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), + session = _make_bridge_session( + key_value="idle-disconnect-quarantine", pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), queued_request_count=0, - last_used_at=2.0, - idle_ttl_seconds=120.0, - ) - service._http_bridge_sessions[key] = stale_session - monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr( - service, - "_create_http_bridge_session", - AsyncMock(return_value=replacement_session), ) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + session.upstream = upstream + session.durable_session_id = "durable-idle-disconnect" + session.durable_owner_epoch = 10 + session.last_completed_response_id = "resp-idle-current-socket" + session.last_completed_response_store = False + session.last_completed_input_count = 2 + session.last_completed_input_prefix_fingerprint = "stored-prefix-proof" + session.last_pending_tool_calls = {"call-idle": "custom_tool_call"} + settings = _make_app_settings(http_responses_session_bridge_instance_id="instance-idle-disconnect") + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + call_order: list[str] = [] + + async def clear_latest_anchor(**kwargs: object) -> object: + del kwargs + call_order.append("quarantine") + return SimpleNamespace( + owner_instance_id="instance-idle-disconnect", + owner_epoch=10, + latest_response_id=None, + ) + + clear = AsyncMock(side_effect=clear_latest_anchor) monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: _make_app_settings(), + service._durable_bridge, + "clear_latest_response_anchor_if_current", + clear, + raising=False, ) - monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-a")) + + async def retry_precreated_request(_session: proxy_service._HTTPBridgeSession) -> bool: + call_order.append("retry") + return False + monkeypatch.setattr( - proxy_service, - "_active_http_bridge_instance_ring", - AsyncMock(return_value=("instance-a", ["instance-a"])), + service, + "_retry_http_bridge_precreated_request", + AsyncMock(side_effect=retry_precreated_request), ) - close_session = AsyncMock() - monkeypatch.setattr(service, "_close_http_bridge_session", close_session) - reused = await service._get_or_create_http_bridge_session( - key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="bridge-key"), - api_key=_make_api_key( - key_id="key-1", - assigned_account_ids=[], - account_assignment_scope_enabled=True, - ), - request_model="gpt-5.4", - idle_ttl_seconds=120.0, - max_sessions=8, - ) + async def fail_reader( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> bool: + del kwargs + call_order.append("settle") + target_session.closed = True + return True - assert reused is replacement_session - assert service._http_bridge_sessions[key] is replacement_session - assert stale_session.closed is True - await _wait_for_close_await(close_session, stale_session) + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + await service._relay_http_bridge_upstream_messages(session) + + clear.assert_awaited_once_with( + session_id="durable-idle-disconnect", + instance_id="instance-idle-disconnect", + owner_epoch=10, + expected_response_id="resp-idle-current-socket", + ) + assert call_order == ["quarantine", "retry", "settle"] + assert not session.pending_requests + assert session.last_completed_response_id is None + assert session.last_completed_response_store is None + assert session.last_pending_tool_calls == {} + assert session.last_completed_input_count == 2 + assert session.last_completed_input_prefix_fingerprint == "stored-prefix-proof" @pytest.mark.asyncio @pytest.mark.parametrize("leading_telemetry", [False, True], ids=["silent", "leading-telemetry"]) +@pytest.mark.parametrize( + ("proxy_injected_anchor", "quarantine_write_result"), + [ + pytest.param(True, "success", id="proxy-injected-anchor"), + pytest.param(True, "error", id="proxy-injected-anchor-write-failure"), + pytest.param(True, "timeout", id="proxy-injected-anchor-write-timeout"), + pytest.param(False, "success", id="client-supplied-anchor"), + ], +) async def test_http_bridge_reader_wakes_and_retires_lone_eventless_owner_without_keepalives( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, leading_telemetry: bool, + proxy_injected_anchor: bool, + quarantine_write_result: str, ) -> None: class _TrackingUpstream: def __init__(self) -> None: @@ -18357,14 +20368,52 @@ async def close(self) -> None: upstream = _TrackingUpstream() session = _make_bridge_session(key_value=f"eventless-{leading_telemetry}") session.upstream = cast(UpstreamWebSocket, upstream) + session.durable_session_id = "durable-eventless-timeout" + session.durable_owner_epoch = 7 service._http_bridge_sessions[session.key] = session settings = _make_app_settings( sse_keepalive_interval_seconds=0.0, stream_idle_timeout_seconds=60.0, http_responses_session_bridge_request_budget_seconds=60.0, http_responses_session_bridge_stuck_gate_retire_after_seconds=0.02, + http_responses_session_bridge_instance_id="instance-eventless-timeout", ) monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + durable_call_order: list[str] = [] + + async def clear_latest_anchor_impl(**kwargs: object) -> None: + del kwargs + durable_call_order.append("clear") + if quarantine_write_result == "error": + raise RuntimeError("durable quarantine unavailable") + if quarantine_write_result == "timeout": + await asyncio.Event().wait() + + if quarantine_write_result == "timeout": + monkeypatch.setattr( + http_bridge_helpers_module, + "_HTTP_BRIDGE_DURABLE_ANCHOR_QUARANTINE_TIMEOUT_SECONDS", + 0.01, + ) + + clear_latest_anchor = AsyncMock(side_effect=clear_latest_anchor_impl) + monkeypatch.setattr( + service._durable_bridge, + "clear_latest_response_anchor_if_current", + clear_latest_anchor, + raising=False, + ) + + async def release_durable_session_impl(**kwargs: object) -> None: + del kwargs + durable_call_order.append("release") + + release_durable_session = AsyncMock(side_effect=release_durable_session_impl) + monkeypatch.setattr( + service._durable_bridge, + "release_live_session", + release_durable_session, + ) retry_precreated = AsyncMock(return_value=False) monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) @@ -18390,6 +20439,8 @@ async def close(self) -> None: owner.request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}' owner.preferred_account_id = "acc-bridge" owner.excluded_account_ids.add("acc-excluded") + owner.previous_response_id = "resp_poisoned" + owner.proxy_injected_previous_response_id = proxy_injected_anchor sibling_queue: asyncio.Queue[str | None] = asyncio.Queue() sibling = proxy_service._WebSocketRequestState( request_id="req-created-sibling", @@ -18450,11 +20501,29 @@ async def close(self) -> None: fail_reader.assert_awaited_once() assert fail_reader.await_args.kwargs["penalize_account"] is False assert fail_reader.await_args.kwargs["force_retire"] is True + release_durable_session.assert_awaited_once_with( + session_id="durable-eventless-timeout", + instance_id="instance-eventless-timeout", + owner_epoch=7, + draining=False, + ) + if proxy_injected_anchor: + clear_latest_anchor.assert_awaited_once_with( + session_id="durable-eventless-timeout", + instance_id="instance-eventless-timeout", + owner_epoch=7, + expected_response_id="resp_poisoned", + ) + else: + clear_latest_anchor.assert_not_awaited() + assert durable_call_order == (["clear", "release"] if proxy_injected_anchor else ["release"]) record_stuck_retire.assert_called_once_with( reason="missing_response_created_timeout", session=session, ) assert "http_bridge_event event=missing_response_created_timeout" in caplog.text + if quarantine_write_result == "timeout": + assert "Timed out quarantining durable HTTP bridge latest-response anchor" in caplog.text @pytest.mark.asyncio @@ -18777,6 +20846,211 @@ async def fail_reader( assert failure_calls[0]["penalize_account"] is True +@pytest.mark.asyncio +async def test_http_bridge_correlated_no_close_failures_are_account_neutral_and_not_replayed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _NoPeerCloseCodexWebSocket: + async def receive(self) -> aiohttp.WSMessage: + return aiohttp.WSMessage( + aiohttp.WSMsgType.CLOSED, + None, + None, + ) + + async def close(self, *, code: int = 1000, message: bytes = b"") -> None: + del code, message + + service = proxy_service.ProxyService(cast(Any, nullcontext())) + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.2, + max_observations=16, + ) + monkeypatch.setattr( + proxy_websocket_module, + "correlate_websocket_egress_failure", + correlator.observe, + raising=False, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + retry_precreated = AsyncMock(return_value=False) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + failure_calls: list[tuple[str, dict[str, object]]] = [] + + async def fail_reader( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> bool: + failure_calls.append((target_session.account.id, dict(kwargs))) + target_session.closed = True + return True + + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + sessions: list[proxy_service._HTTPBridgeSession] = [] + request_states: list[proxy_service._WebSocketRequestState] = [] + for account_id in ("acc-correlated-a", "acc-correlated-b"): + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-{account_id}", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + transport="http", + ) + session = _make_bridge_session( + key_value=f"bridge-{account_id}", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.account = cast( + Any, + SimpleNamespace(id=account_id, status=AccountStatus.ACTIVE, plan_type="plus"), + ) + session.upstream = CodexUpstreamWebSocket( + _NoPeerCloseCodexWebSocket(), + correlate_no_close_failures=True, + account_id=account_id, + egress_key="routed_proxy:bridge-shared-endpoint", + ) + sessions.append(session) + request_states.append(request_state) + + await asyncio.gather( + *(service._relay_http_bridge_upstream_messages(session) for session in sessions), + ) + + assert sorted(account_id for account_id, _kwargs in failure_calls) == [ + "acc-correlated-a", + "acc-correlated-b", + ] + assert all(kwargs["error_code"] == "proxy_network_unavailable" for _account_id, kwargs in failure_calls) + assert all(kwargs["penalize_account"] is False for _account_id, kwargs in failure_calls) + assert all(session.last_upstream_close_code is None for session in sessions) + assert all(request_state.replay_count == 0 for request_state in request_states) + retry_precreated.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_deadline_waits_for_inflight_no_close_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _NoPeerCloseCodexWebSocket: + async def receive(self) -> aiohttp.WSMessage: + return aiohttp.WSMessage( + aiohttp.WSMsgType.CLOSED, + None, + None, + ) + + async def close(self, *, code: int = 1000, message: bytes = b"") -> None: + del code, message + + service = proxy_service.ProxyService(cast(Any, nullcontext())) + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.1, + max_observations=16, + ) + first_observed = asyncio.Event() + + async def observe( + *, + egress_key: str | None, + account_id: str | None, + wait_for_correlation: bool = True, + ) -> bool: + if account_id == "acc-bridge-deadline-a": + first_observed.set() + return await correlator.observe( + egress_key=egress_key, + account_id=account_id, + wait_for_correlation=wait_for_correlation, + ) + + monkeypatch.setattr( + proxy_websocket_module, + "correlate_websocket_egress_failure", + observe, + raising=False, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=0.01, + stream_idle_timeout_seconds=5.0, + ), + ) + retry_precreated = AsyncMock(return_value=False) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + failure_calls: list[tuple[str, dict[str, object]]] = [] + + async def fail_reader( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> bool: + failure_calls.append((target_session.account.id, dict(kwargs))) + target_session.closed = True + return True + + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + def make_session(account_id: str) -> proxy_service._HTTPBridgeSession: + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-{account_id}", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + transport="http", + ) + session = _make_bridge_session( + key_value=f"bridge-{account_id}", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.last_used_at = time.monotonic() + session.account = cast( + Any, + SimpleNamespace(id=account_id, status=AccountStatus.ACTIVE, plan_type="plus"), + ) + session.upstream = CodexUpstreamWebSocket( + _NoPeerCloseCodexWebSocket(), + correlate_no_close_failures=True, + account_id=account_id, + egress_key="routed_proxy:bridge-deadline-shared", + ) + return session + + first_relay = asyncio.create_task( + service._relay_http_bridge_upstream_messages(make_session("acc-bridge-deadline-a")) + ) + await asyncio.wait_for(first_observed.wait(), timeout=0.1) + await asyncio.sleep(0.03) + assert not first_relay.done() + + second_relay = asyncio.create_task( + service._relay_http_bridge_upstream_messages(make_session("acc-bridge-deadline-b")) + ) + await asyncio.wait_for( + asyncio.gather(first_relay, second_relay), + timeout=0.5, + ) + + assert sorted(account_id for account_id, _kwargs in failure_calls) == [ + "acc-bridge-deadline-a", + "acc-bridge-deadline-b", + ] + assert all(kwargs["error_code"] == "proxy_network_unavailable" for _account_id, kwargs in failure_calls) + assert all(kwargs["penalize_account"] is False for _account_id, kwargs in failure_calls) + retry_precreated.assert_not_awaited() + + @pytest.mark.asyncio async def test_http_bridge_response_create_gate_timeout_logs_pending_bridge_context( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 31910f68ca..d9241b2e2b 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -31,6 +31,7 @@ from websockets.frames import Close import app.core.clients.proxy as proxy_module +import app.core.clients.proxy_websocket as proxy_websocket_module import app.core.resilience.network_recovery as network_recovery_module import app.modules.proxy.load_balancer as load_balancer_module from app.core.balancer.types import UpstreamError @@ -15283,6 +15284,130 @@ async def fake_reconnect_http_bridge_session( assert request_state.event_queue.empty() +@pytest.mark.asyncio +async def test_http_bridge_account_bound_token_invalidated_never_fails_over(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + owner_account = _make_account("acc_bridge_compaction_owner") + request_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.1", + "input": [ + { + "id": "cmp_account_bound_auth", + "type": "compaction", + "encrypted_content": "encrypted-complete-context", + } + ], + }, + separators=(",", ":"), + ) + + class _FakeUpstreamWebSocket: + def __init__(self) -> None: + self.sent_text: list[str] = [] + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + + retry_upstream = _FakeUpstreamWebSocket() + reconnect_calls: list[dict[str, object]] = [] + + async def fake_reconnect_http_bridge_session( + session, + *, + request_state, + restart_reader=False, + require_security_work_authorized=False, + require_same_account=False, + require_preferred_account=False, + ): + reconnect_calls.append( + { + "owner_id": request_state.account_bound_owner_id, + "preferred_account_id": request_state.preferred_account_id, + "excluded_account_ids": set(request_state.excluded_account_ids), + "restart_reader": restart_reader, + "require_security_work_authorized": require_security_work_authorized, + "require_same_account": require_same_account, + "require_preferred_account": require_preferred_account, + } + ) + session.account = owner_account + session.upstream = retry_upstream + session.upstream_control = proxy_service._WebSocketUpstreamControl() + + mark_permanent_failure = AsyncMock() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", fake_reconnect_http_bridge_session) + monkeypatch.setattr(service._load_balancer, "mark_permanent_failure", mark_permanent_failure) + + request_state = proxy_service._WebSocketRequestState( + request_id="bridge_req_compaction_token_invalidated", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + request_text=request_text, + preferred_account_id=owner_account.id, + account_bound_owner_id=owner_account.id, + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey( + "turn_state_header", + "turn-compaction-token-invalidated", + None, + ), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.1", + account=owner_account, + upstream=cast(proxy_service.UpstreamWebSocket, _FakeUpstreamWebSocket()), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=1.0, + idle_ttl_seconds=300.0, + ) + token_invalidated_text = json.dumps( + { + "type": "error", + "error": { + "code": "token_invalidated", + "type": "invalid_request_error", + "message": "Your authentication token has been invalidated. Please try signing in again.", + }, + }, + separators=(",", ":"), + ) + + await service._process_http_bridge_upstream_text(session, token_invalidated_text) + await service._process_http_bridge_upstream_text(session, token_invalidated_text) + + assert reconnect_calls == [ + { + "owner_id": owner_account.id, + "preferred_account_id": owner_account.id, + "excluded_account_ids": set(), + "restart_reader": False, + "require_security_work_authorized": False, + "require_same_account": True, + "require_preferred_account": False, + } + ] + mark_permanent_failure.assert_awaited_once_with(owner_account, "account_auth_invalidated") + assert retry_upstream.sent_text == [request_text] + assert session.account is owner_account + assert request_state.account_bound_owner_id == owner_account.id + assert request_state.excluded_account_ids == {owner_account.id} + + @pytest.mark.asyncio async def test_http_bridge_nonreplayable_auth_failure_marks_account_permanent(monkeypatch): request_logs = _RequestLogsRecorder() @@ -25413,6 +25538,309 @@ async def close(self) -> None: assert terminal["response"]["error"]["code"] == "proxy_network_unavailable" +@pytest.mark.asyncio +async def test_relay_upstream_websocket_correlated_no_close_incident_skips_account_health_for_all_accounts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _FakeDownstreamWebSocket: + def __init__(self) -> None: + self.sent_text: list[str] = [] + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + + class _NoCloseConnection: + async def recv(self) -> str: + raise ConnectionClosedError(None, None) + + async def close(self, code: int = 1000, reason: str = "") -> None: + del code, reason + + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + handle_stream_error = AsyncMock() + monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error) + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", AsyncMock()) + correlator = network_recovery_module.WebSocketEgressFailureCorrelator( + window_seconds=0.2, + max_observations=16, + ) + monkeypatch.setattr( + proxy_websocket_module, + "correlate_websocket_egress_failure", + correlator.observe, + raising=False, + ) + + relays: list[Any] = [] + request_states: list[proxy_service._WebSocketRequestState] = [] + upstream_controls: list[proxy_service._WebSocketUpstreamControl] = [] + downstreams: list[_FakeDownstreamWebSocket] = [] + for account_id in ("acc-ws-correlated-a", "acc-ws-correlated-b"): + account = _make_account(account_id) + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-{account_id}", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + awaiting_response_created=True, + downstream_visible=True, + ) + pending_requests = deque([request_state]) + upstream_control = proxy_service._WebSocketUpstreamControl() + downstream = _FakeDownstreamWebSocket() + upstream = WebsocketsUpstreamWebSocket( + cast(Any, _NoCloseConnection()), + correlate_no_close_failures=True, + account_id=account_id, + egress_key="environment_proxy:http://direct-relay.shared.test:7890", + ) + relays.append( + service._relay_upstream_websocket_messages( + cast(WebSocket, downstream), + upstream, + account=account, + account_id_value=account_id, + pending_requests=pending_requests, + pending_lock=anyio.Lock(), + client_send_lock=anyio.Lock(), + api_key=None, + upstream_control=upstream_control, + response_create_gate=asyncio.Semaphore(1), + proxy_request_budget_seconds=5.0, + stream_idle_timeout_seconds=5.0, + downstream_activity=proxy_service._DownstreamWebSocketActivity(), + ) + ) + request_states.append(request_state) + upstream_controls.append(upstream_control) + downstreams.append(downstream) + + await asyncio.gather(*relays) + + handle_stream_error.assert_not_awaited() + assert all(request_state.replay_count == 0 for request_state in request_states) + assert all(control.reconnect_requested is False for control in upstream_controls) + assert [json.loads(downstream.sent_text[-1])["response"]["error"]["code"] for downstream in downstreams] == [ + "proxy_network_unavailable", + "proxy_network_unavailable", + ] + + +@pytest.mark.asyncio +async def test_relay_upstream_websocket_no_close_correlation_survives_short_keepalive( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _FakeDownstreamWebSocket: + def __init__(self) -> None: + self.sent_text: list[str] = [] + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + + class _NoCloseConnection: + async def recv(self) -> str: + raise ConnectionClosedError(None, None) + + async def close(self, code: int = 1000, reason: str = "") -> None: + del code, reason + + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + handle_stream_error = AsyncMock() + monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error) + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", AsyncMock()) + settings = _make_proxy_settings() + settings.sse_keepalive_interval_seconds = 0.01 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + correlator = network_recovery_module.WebSocketEgressFailureCorrelator( + window_seconds=0.05, + max_observations=16, + ) + monkeypatch.setattr( + proxy_websocket_module, + "correlate_websocket_egress_failure", + correlator.observe, + raising=False, + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_short_keepalive", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + awaiting_response_created=True, + downstream_visible=True, + ) + pending_requests = deque([request_state]) + downstream = _FakeDownstreamWebSocket() + account = _make_account("acc-ws-short-keepalive") + upstream = WebsocketsUpstreamWebSocket( + cast(Any, _NoCloseConnection()), + correlate_no_close_failures=True, + account_id=account.id, + egress_key="environment_proxy:http://short-keepalive.test:7890", + ) + + await asyncio.wait_for( + service._relay_upstream_websocket_messages( + cast(WebSocket, downstream), + upstream, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=anyio.Lock(), + client_send_lock=anyio.Lock(), + api_key=None, + upstream_control=proxy_service._WebSocketUpstreamControl(), + response_create_gate=asyncio.Semaphore(1), + proxy_request_budget_seconds=1.0, + stream_idle_timeout_seconds=1.0, + downstream_activity=proxy_service._DownstreamWebSocketActivity(), + ), + timeout=0.5, + ) + + assert list(pending_requests) == [] + terminal = json.loads(downstream.sent_text[-1]) + assert terminal["response"]["error"]["code"] == "stream_incomplete" + handle_stream_error.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_relay_upstream_websocket_deadline_waits_for_inflight_no_close_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _FakeDownstreamWebSocket: + def __init__(self) -> None: + self.sent_text: list[str] = [] + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + + class _NoCloseConnection: + async def recv(self) -> str: + raise ConnectionClosedError(None, None) + + async def close(self, code: int = 1000, reason: str = "") -> None: + del code, reason + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + handle_stream_error = AsyncMock() + monkeypatch.setattr(service, "_handle_stream_error", handle_stream_error) + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", AsyncMock()) + correlator = network_recovery_module.WebSocketEgressFailureCorrelator( + window_seconds=0.3, + max_observations=16, + ) + first_observed = asyncio.Event() + + async def observe( + *, + egress_key: str | None, + account_id: str | None, + wait_for_correlation: bool = True, + ) -> bool: + if account_id == "acc-ws-deadline-a": + first_observed.set() + return await correlator.observe( + egress_key=egress_key, + account_id=account_id, + wait_for_correlation=wait_for_correlation, + ) + + monkeypatch.setattr( + proxy_websocket_module, + "correlate_websocket_egress_failure", + observe, + raising=False, + ) + + def start_relay( + account_id: str, + *, + request_budget_seconds: float, + ) -> tuple[ + asyncio.Task[None], + proxy_service._WebSocketRequestState, + _FakeDownstreamWebSocket, + ]: + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-{account_id}", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + request_text='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + awaiting_response_created=True, + downstream_visible=True, + ) + downstream = _FakeDownstreamWebSocket() + upstream = WebsocketsUpstreamWebSocket( + cast(Any, _NoCloseConnection()), + correlate_no_close_failures=True, + account_id=account_id, + egress_key="environment_proxy:http://deadline.shared.test:7890", + ) + relay = asyncio.create_task( + service._relay_upstream_websocket_messages( + cast(WebSocket, downstream), + upstream, + account=_make_account(account_id), + account_id_value=account_id, + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + client_send_lock=anyio.Lock(), + api_key=None, + upstream_control=proxy_service._WebSocketUpstreamControl(), + response_create_gate=asyncio.Semaphore(1), + proxy_request_budget_seconds=request_budget_seconds, + stream_idle_timeout_seconds=5.0, + downstream_activity=proxy_service._DownstreamWebSocketActivity(), + ) + ) + return relay, request_state, downstream + + first_relay, first_state, first_downstream = start_relay( + "acc-ws-deadline-a", + request_budget_seconds=0.1, + ) + await asyncio.wait_for(first_observed.wait(), timeout=0.1) + await asyncio.sleep(0.15) + assert not first_relay.done() + + second_relay, second_state, second_downstream = start_relay( + "acc-ws-deadline-b", + request_budget_seconds=1.0, + ) + await asyncio.wait_for( + asyncio.gather(first_relay, second_relay), + timeout=0.75, + ) + + handle_stream_error.assert_not_awaited() + assert first_state.replay_count == 0 + assert second_state.replay_count == 0 + assert json.loads(first_downstream.sent_text[-1])["response"]["error"]["code"] == ("proxy_network_unavailable") + assert json.loads(second_downstream.sent_text[-1])["response"]["error"]["code"] == ("proxy_network_unavailable") + + @pytest.mark.asyncio @pytest.mark.parametrize("routed", [False, True], ids=["direct-close", "routed-receive-error"]) async def test_relay_upstream_websocket_ordinary_receive_failure_is_stream_incomplete_and_penalized( @@ -37996,6 +38424,121 @@ async def capture_send_text(_text: str) -> None: assert list(session.pending_requests) == [request_state] +@pytest.mark.asyncio +async def test_submit_http_bridge_stale_anchor_fallback_keeps_external_images_inlined( + monkeypatch: pytest.MonkeyPatch, +) -> None: + external_url = "https://example.com/fallback.png" + data_url = "data:image/png;base64,aW5saW5lZA==" + anchored_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.5", + "previous_response_id": "resp-old-socket", + "input": [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": external_url}], + } + ], + }, + ensure_ascii=True, + separators=(",", ":"), + ) + fresh_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.5", + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "earlier"}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "context"}]}, + { + "role": "user", + "content": [{"type": "input_image", "image_url": external_url}], + }, + ], + }, + ensure_ascii=True, + separators=(",", ":"), + ) + + async def fake_inline(payload_dict, _session, _timeout): + return json.loads(json.dumps(payload_dict).replace(external_url, data_url)) + + class FakeSettings: + image_inline_fetch_enabled = True + upstream_connect_timeout_seconds = 5.0 + + class FakeSession: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + monkeypatch.setattr(proxy_service, "get_settings", lambda: FakeSettings()) + monkeypatch.setattr(proxy_service, "_inline_input_image_urls", fake_inline) + monkeypatch.setattr(proxy_service, "lease_http_session", lambda: FakeSession()) + monkeypatch.setattr(proxy_service, "_as_image_fetch_session", lambda session: session) + + service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + request_state = proxy_service._WebSocketRequestState( + request_id="req-stale-anchor-image-fallback", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text=anchored_text, + previous_response_id="resp-old-socket", + response_store=False, + proxy_injected_previous_response_id=True, + fresh_upstream_request_text=fresh_text, + fresh_upstream_request_is_retry_safe=True, + archive_request_id="archive-stale-anchor-image-fallback", + ) + send_text = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-stale-anchor-image", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-stale-anchor-image"), + request_model="gpt-5.5", + account=cast(Account, SimpleNamespace(id="acc-stale-anchor-image")), + upstream=cast( + proxy_service.UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), + ), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + ) + + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", AsyncMock()) + monkeypatch.setattr(service, "_start_request_state_api_key_reservation_heartbeat", lambda *args, **kwargs: None) + + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=anchored_text, + queue_limit=1, + ) + + assert send_text.await_args is not None + sent_text = send_text.await_args.args[0] + assert external_url not in sent_text + assert data_url in sent_text + assert request_state.previous_response_id is None + assert request_state.proxy_injected_previous_response_id is False + assert request_state.fresh_upstream_request_text == sent_text + + @pytest.mark.asyncio async def test_submit_http_bridge_network_send_failure_is_neutral_and_not_replayed(monkeypatch): service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) diff --git a/tests/unit/test_proxy_websocket_client.py b/tests/unit/test_proxy_websocket_client.py index 034d0ca6c2..3022ed6460 100644 --- a/tests/unit/test_proxy_websocket_client.py +++ b/tests/unit/test_proxy_websocket_client.py @@ -16,6 +16,7 @@ from websockets.http11 import Response import app.core.clients.proxy_websocket as proxy_websocket_module +import app.core.resilience.network_recovery as network_recovery from app.core.clients.codex import CodexTransportError, CodexWebSocketResult from app.core.clients.proxy import ProxyResponseError from app.core.clients.proxy_websocket import ( @@ -338,6 +339,449 @@ async def recv(self) -> str: assert all(call.kwargs["transport"] == "websocket" for call in rotate.await_args_list) +def test_websocket_egress_key_distinguishes_route_environment_proxy_and_direct_without_credentials() -> None: + url = "wss://chatgpt.com/backend-api/codex/responses" + proxy_url = "http://proxy-user:proxy-secret@proxy.shared.test:7890" + + routed = proxy_websocket_module._websocket_egress_key( + url, + route_endpoint_id="ep-shared", + ) + environment_proxy = proxy_websocket_module._websocket_egress_key( + url, + proxy_url=proxy_url, + ) + direct = proxy_websocket_module._websocket_egress_key(url) + + assert routed == "routed_proxy:ep-shared" + assert environment_proxy == "environment_proxy:http://proxy.shared.test:7890" + assert direct == "direct:wss://chatgpt.com:443" + assert "proxy-user" not in str((routed, environment_proxy, direct)) + assert "proxy-secret" not in str((routed, environment_proxy, direct)) + + +@pytest.mark.asyncio +async def test_responses_env_proxy_no_close_failures_correlate_across_accounts_without_leaking_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _NoCloseConnection(_FakeConnection): + async def recv(self) -> str: + raise ConnectionClosedError(None, None) + + class _Settings(SimpleNamespace): + def upstream_websocket_proxy_env(self) -> dict[str, str]: + return { + "https_proxy": "http://proxy-user:proxy-secret@proxy.shared.test:7890", + } + + connections = [_NoCloseConnection(), _NoCloseConnection()] + connector = AsyncMock(side_effect=connections) + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.2, + max_observations=16, + ) + observations: list[tuple[str | None, str | None]] = [] + + async def observe( + *, + egress_key: str | None, + account_id: str | None, + wait_for_correlation: bool = True, + ) -> bool: + observations.append((egress_key, account_id)) + return await correlator.observe( + egress_key=egress_key, + account_id=account_id, + wait_for_correlation=wait_for_correlation, + ) + + rotate = AsyncMock(return_value="rotated") + monkeypatch.setattr(proxy_websocket_module, "websocket_connect", connector) + monkeypatch.setattr(proxy_websocket_module, "correlate_websocket_egress_failure", observe, raising=False) + monkeypatch.setattr(proxy_websocket_module, "rotate_shared_http_transport", rotate) + monkeypatch.setattr( + proxy_websocket_module, + "get_settings", + lambda: _Settings( + upstream_base_url="https://chatgpt.com/backend-api", + upstream_connect_timeout_seconds=7.0, + max_sse_event_bytes=4321, + upstream_websocket_trust_env=True, + ), + ) + + first = await connect_responses_websocket( + {}, + "token-a", + "account-a", + allow_direct_egress=True, + ) + second = await connect_responses_websocket( + {}, + "token-b", + "account-b", + allow_direct_egress=True, + ) + messages = await asyncio.gather(first.receive(), second.receive()) + + assert [message.error_code for message in messages] == [ + "proxy_network_unavailable", + "proxy_network_unavailable", + ] + assert observations == [ + ("environment_proxy:http://proxy.shared.test:7890", "account-a"), + ("environment_proxy:http://proxy.shared.test:7890", "account-b"), + ] + assert "proxy-user" not in str(observations) + assert "proxy-secret" not in str(observations) + rotate.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_responses_known_network_failure_correlates_prior_ambiguous_account_without_waiting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _ReceiveFailureConnection(_FakeConnection): + def __init__(self, error: BaseException) -> None: + super().__init__() + self._error = error + + async def recv(self) -> str: + raise self._error + + connector = AsyncMock( + side_effect=[ + _ReceiveFailureConnection(ConnectionClosedError(None, None)), + _ReceiveFailureConnection(OSError(errno.ENETUNREACH, "Network is unreachable")), + ] + ) + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.5, + max_observations=16, + ) + first_observed = asyncio.Event() + observations: list[tuple[str | None, str | None, bool]] = [] + + async def observe( + *, + egress_key: str | None, + account_id: str | None, + wait_for_correlation: bool = True, + ) -> bool: + observations.append((egress_key, account_id, wait_for_correlation)) + if account_id == "account-a": + first_observed.set() + return await correlator.observe( + egress_key=egress_key, + account_id=account_id, + wait_for_correlation=wait_for_correlation, + ) + + monkeypatch.setattr(proxy_websocket_module, "websocket_connect", connector) + monkeypatch.setattr(proxy_websocket_module, "correlate_websocket_egress_failure", observe, raising=False) + monkeypatch.setattr(proxy_websocket_module, "rotate_shared_http_transport", AsyncMock(return_value="rotated")) + monkeypatch.setattr( + proxy_websocket_module, + "get_settings", + lambda: SimpleNamespace( + upstream_base_url="https://chatgpt.com/backend-api", + upstream_connect_timeout_seconds=7.0, + max_sse_event_bytes=4321, + upstream_websocket_trust_env=False, + ), + ) + + first = await connect_responses_websocket( + {}, + "token-a", + "account-a", + allow_direct_egress=True, + ) + second = await connect_responses_websocket( + {}, + "token-b", + "account-b", + allow_direct_egress=True, + ) + first_receive = asyncio.create_task(first.receive()) + await first_observed.wait() + + second_message = await asyncio.wait_for(second.receive(), timeout=0.1) + first_message = await asyncio.wait_for(first_receive, timeout=0.1) + + assert [first_message.error_code, second_message.error_code] == [ + "proxy_network_unavailable", + "proxy_network_unavailable", + ] + assert observations == [ + ("direct:wss://chatgpt.com:443", "account-a", True), + ("direct:wss://chatgpt.com:443", "account-b", False), + ] + + +@pytest.mark.asyncio +async def test_routed_responses_error_uses_actual_endpoint_for_cross_account_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requested_route = ResolvedUpstreamRoute( + mode="account_bound", + pool_id="pool-requested", + endpoint=ResolvedProxyEndpoint("ep-requested", "http", "requested.proxy.test", 8080), + ) + actual_route = ResolvedUpstreamRoute( + mode="account_bound", + pool_id="pool-actual", + endpoint=ResolvedProxyEndpoint("ep-actual", "http", "actual.proxy.test", 8080), + ) + + class _ActualRouteCodexClient(_FakeCodexClient): + async def open_ws_with_route_metadata( + self, + url: str, + *, + route: ResolvedUpstreamRoute, + **kwargs: object, + ) -> CodexWebSocketResult: + self.calls.append({"url": url, "route": route, **kwargs}) + return CodexWebSocketResult( + websocket=self.websocket, + context=None, + route=actual_route, + fallback_used=True, + ) + + correlator = network_recovery.WebSocketEgressFailureCorrelator( + window_seconds=0.2, + max_observations=16, + ) + observations: list[tuple[str | None, str | None]] = [] + + async def observe( + *, + egress_key: str | None, + account_id: str | None, + wait_for_correlation: bool = True, + ) -> bool: + observations.append((egress_key, account_id)) + return await correlator.observe( + egress_key=egress_key, + account_id=account_id, + wait_for_correlation=wait_for_correlation, + ) + + monkeypatch.setattr(proxy_websocket_module, "correlate_websocket_egress_failure", observe) + monkeypatch.setattr( + proxy_websocket_module, + "get_settings", + lambda: SimpleNamespace( + upstream_base_url="https://chatgpt.com/backend-api", + upstream_connect_timeout_seconds=7.0, + max_sse_event_bytes=4321, + upstream_websocket_trust_env=False, + ), + ) + first = await connect_responses_websocket( + {}, + "token-a", + "account-a", + route=requested_route, + codex_client=cast( + Any, + _ActualRouteCodexClient(_FakeCodexErrorWebSocket(ConnectionResetError("reset-a"))), + ), + ) + second = await connect_responses_websocket( + {}, + "token-b", + "account-b", + route=requested_route, + codex_client=cast( + Any, + _ActualRouteCodexClient(_FakeCodexErrorWebSocket(ConnectionResetError("reset-b"))), + ), + ) + + messages = await asyncio.gather(first.receive(), second.receive()) + + assert [message.error_code for message in messages] == [ + "proxy_network_unavailable", + "proxy_network_unavailable", + ] + assert observations == [ + ("routed_proxy:ep-actual", "account-a"), + ("routed_proxy:ep-actual", "account-b"), + ] + + +@pytest.mark.asyncio +async def test_responses_explicit_close_frame_does_not_enter_no_close_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _ExplicitCloseConnection(_FakeConnection): + async def recv(self) -> str: + raise ConnectionClosedError(Close(1011, "server restart"), None) + + correlate = AsyncMock(return_value=True) + monkeypatch.setattr(proxy_websocket_module, "websocket_connect", AsyncMock(return_value=_ExplicitCloseConnection())) + monkeypatch.setattr( + proxy_websocket_module, + "correlate_websocket_egress_failure", + correlate, + raising=False, + ) + monkeypatch.setattr( + proxy_websocket_module, + "get_settings", + lambda: SimpleNamespace( + upstream_base_url="https://chatgpt.com/backend-api", + upstream_connect_timeout_seconds=7.0, + max_sse_event_bytes=4321, + upstream_websocket_trust_env=False, + ), + ) + + websocket = await connect_responses_websocket( + {}, + "token-a", + "account-a", + allow_direct_egress=True, + ) + message = await websocket.receive() + + assert message.kind == "error" + assert message.close_code == 1011 + assert message.error_code is None + correlate.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_routed_responses_explicit_close_message_and_followup_closed_do_not_enter_no_close_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _ExplicitCloseCodexWebSocket(_FakeCodexWebSocket): + def __init__(self) -> None: + super().__init__() + self.messages = iter( + ( + aiohttp.WSMessage( + aiohttp.WSMsgType.CLOSE, + 1011, + "server restart", + ), + aiohttp.WSMessage( + aiohttp.WSMsgType.CLOSED, + None, + None, + ), + ) + ) + + async def receive(self) -> aiohttp.WSMessage: + return next(self.messages) + + route = ResolvedUpstreamRoute( + mode="account_bound", + pool_id="pool-close", + endpoint=ResolvedProxyEndpoint("ep-close", "http", "close.proxy.test", 8080), + ) + correlate = AsyncMock(return_value=True) + monkeypatch.setattr(proxy_websocket_module, "correlate_websocket_egress_failure", correlate) + monkeypatch.setattr( + proxy_websocket_module, + "get_settings", + lambda: SimpleNamespace( + upstream_base_url="https://chatgpt.com/backend-api", + upstream_connect_timeout_seconds=7.0, + max_sse_event_bytes=4321, + upstream_websocket_trust_env=False, + ), + ) + + upstream = await connect_responses_websocket( + {}, + "token-a", + "account-a", + route=route, + codex_client=cast(Any, _FakeCodexClient(_ExplicitCloseCodexWebSocket())), + ) + message = await upstream.receive() + closed_message = await upstream.receive() + + assert message.kind == "close" + assert message.close_code == 1011 + assert closed_message.kind == "close" + correlate.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_routed_non_responses_closed_message_preserves_existing_close_semantics( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _ClosedCodexWebSocket(_FakeCodexWebSocket): + async def receive(self) -> aiohttp.WSMessage: + return aiohttp.WSMessage( + aiohttp.WSMsgType.CLOSED, + None, + None, + ) + + correlate = AsyncMock(return_value=True) + monkeypatch.setattr(proxy_websocket_module, "correlate_websocket_egress_failure", correlate) + upstream = CodexUpstreamWebSocket( + _ClosedCodexWebSocket(), + correlate_no_close_failures=False, + account_id="account-live", + egress_key="routed_proxy:live-endpoint", + ) + + message = await upstream.receive() + + assert message.kind == "close" + correlate.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_live_no_close_failure_does_not_enter_responses_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _NoCloseConnection(_FakeConnection): + async def recv(self) -> str: + raise ConnectionClosedError(None, None) + + correlate = AsyncMock(return_value=True) + monkeypatch.setattr(proxy_websocket_module, "websocket_connect", AsyncMock(return_value=_NoCloseConnection())) + monkeypatch.setattr( + proxy_websocket_module, + "correlate_websocket_egress_failure", + correlate, + raising=False, + ) + monkeypatch.setattr( + proxy_websocket_module, + "get_settings", + lambda: SimpleNamespace( + upstream_connect_timeout_seconds=7.0, + proxy_downstream_websocket_idle_timeout_seconds=120.0, + max_sse_event_bytes=4321, + upstream_websocket_trust_env=False, + ), + ) + + websocket = await connect_live_websocket( + "rtc_no_close", + {}, + "token-a", + "account-a", + protocol=RealtimeWebSocketProtocol.LIVE_V3, + allow_direct_egress=True, + ) + message = await websocket.receive() + + assert message.kind == "error" + assert message.error_code is None + correlate.assert_not_awaited() + + @pytest.mark.asyncio async def test_connect_responses_websocket_routed_codex_call_preserves_size_limit(monkeypatch): route = ResolvedUpstreamRoute( diff --git a/tests/unit/test_replay_safety.py b/tests/unit/test_replay_safety.py index a281ad1835..b9807eb795 100644 --- a/tests/unit/test_replay_safety.py +++ b/tests/unit/test_replay_safety.py @@ -10,9 +10,11 @@ ) from app.modules.proxy.replay_safety import ( project_responses_input_for_account_neutral_fresh_replay, + responses_input_has_self_contained_tool_continuation_suffix, responses_input_suffix_matches_pending_tool_calls, responses_input_suffix_retains_prior_output, responses_payload_is_account_neutral_fresh_replay, + responses_payload_is_same_account_compaction_recovery, ) @@ -151,6 +153,115 @@ def test_account_neutral_fresh_replay_accepts_self_contained_payloads( assert responses_payload_is_account_neutral_fresh_replay(payload) is True +@pytest.mark.parametrize( + ("payload", "expected"), + [ + pytest.param( + { + "model": "gpt-5.4", + "instructions": "continue", + "input": [{"id": "cmp_valid", "type": "compaction", "encrypted_content": "ciphertext"}], + }, + True, + id="exact-compaction", + ), + pytest.param( + { + "input": [ + {"id": "cmp_valid", "type": "compaction", "encrypted_content": "ciphertext"}, + {"type": "message", "role": "user", "content": "continue"}, + ] + }, + True, + id="account-neutral-suffix", + ), + pytest.param( + {"input": [{"type": "compaction", "encrypted_content": "ciphertext"}]}, + False, + id="missing-id", + ), + pytest.param( + {"input": [{"id": " ", "type": "compaction", "encrypted_content": "ciphertext"}]}, + False, + id="blank-id", + ), + pytest.param( + {"input": [{"id": "cmp_empty", "type": "compaction", "encrypted_content": ""}]}, + False, + id="empty-ciphertext", + ), + pytest.param( + { + "input": [ + { + "id": "cmp_extra", + "type": "compaction", + "encrypted_content": "ciphertext", + "summary": "untrusted", + } + ] + }, + False, + id="unknown-compaction-field", + ), + pytest.param( + {"input": [{"type": "message", "role": "system", "content": "summary of the old session"}]}, + False, + id="plaintext-summary", + ), + pytest.param( + { + "input": [ + {"id": "cmp_file", "type": "compaction", "encrypted_content": "ciphertext"}, + {"type": "input_file", "file_id": "file_owner_scoped"}, + ] + }, + False, + id="account-scoped-suffix", + ), + pytest.param( + { + "input": [ + {"id": "cmp_nested", "type": "compaction", "encrypted_content": "ciphertext"}, + {"type": "reasoning", "encrypted_content": "second-ciphertext"}, + ] + }, + False, + id="second-encrypted-item", + ), + pytest.param( + { + "previous_response_id": "resp_explicit", + "input": [{"id": "cmp_explicit", "type": "compaction", "encrypted_content": "ciphertext"}], + }, + False, + id="explicit-anchor", + ), + pytest.param( + { + "conversation": "conv_explicit", + "input": [{"id": "cmp_conversation", "type": "compaction", "encrypted_content": "ciphertext"}], + }, + False, + id="explicit-conversation", + ), + pytest.param( + { + "input": [{"id": "cmp_unknown", "type": "compaction", "encrypted_content": "ciphertext"}], + "future_state": {"id": "owner-scoped"}, + }, + False, + id="unknown-request-state", + ), + ], +) +def test_same_account_compaction_recovery_requires_exact_shape_and_neutral_suffix( + payload: dict[str, JsonValue], + expected: bool, +) -> None: + assert responses_payload_is_same_account_compaction_recovery(payload) is expected + + def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() -> None: metadata = {"turn_id": "turn_owner_a"} input_items: list[JsonValue] = [ @@ -593,6 +704,268 @@ def test_full_resend_suffix_rejects_missing_or_misordered_context( ) +@pytest.mark.parametrize( + "suffix", + [ + pytest.param( + [ + { + "type": "custom_tool_call", + "call_id": "call_1", + "name": "exec", + "input": "pwd", + "status": "completed", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_1", + "output": "/workspace", + "status": "completed", + }, + ], + id="immediate-custom-tool-continuation-without-new-user", + ), + pytest.param( + [ + { + "type": "message", + "role": "assistant", + "phase": "commentary", + "content": [{"type": "output_text", "text": "Checking the workspace."}], + }, + { + "type": "custom_tool_call", + "call_id": "call_2", + "name": "exec", + "input": "git status", + "status": "completed", + "caller": {"type": "direct"}, + }, + { + "type": "custom_tool_call_output", + "call_id": "call_2", + "output": "clean", + "status": "completed", + "caller": {"type": "direct"}, + }, + {"type": "message", "role": "developer", "content": "continue safely"}, + {"type": "message", "role": "user", "content": "continue"}, + ], + id="observed-codex-full-history-shape", + ), + pytest.param( + [ + { + "type": "function_call", + "call_id": "call_3", + "name": "lookup", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "call_3", "output": "result"}, + ], + id="immediate-function-tool-continuation", + ), + ], +) +def test_quarantine_tool_continuation_accepts_only_complete_self_contained_suffix( + suffix: list[JsonValue], +) -> None: + stored_input: list[JsonValue] = [{"role": "user", "content": "first question"}] + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + + assert projection is not None + assert ( + responses_input_has_self_contained_tool_continuation_suffix( + projection.input_items, + stored_count=projection.stored_prefix_count, + ) + is True + ) + + +def test_quarantine_tool_continuation_keeps_existing_tool_declarations_on_same_account() -> None: + stored_input: list[JsonValue] = [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "mcp", "server_label": "workspace-tools"}], + }, + {"role": "user", "content": "first question"}, + ] + suffix: list[JsonValue] = [ + { + "type": "custom_tool_call", + "call_id": "call_1", + "name": "exec", + "input": "pwd", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_1", + "output": "/workspace", + }, + ] + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + + assert projection is not None + assert responses_payload_is_account_neutral_fresh_replay({"input": projection.input_items}) is False + assert ( + responses_input_has_self_contained_tool_continuation_suffix( + projection.input_items, + stored_count=projection.stored_prefix_count, + ) + is True + ) + + +@pytest.mark.parametrize( + ("stored_input", "suffix"), + [ + pytest.param( + [{"role": "user", "content": "first question"}], + [{"type": "custom_tool_call_output", "call_id": "call_1", "output": "orphan"}], + id="orphan-output", + ), + pytest.param( + [{"role": "user", "content": "first question"}], + [ + { + "type": "custom_tool_call", + "call_id": "call_1", + "name": "exec", + "input": "pwd", + } + ], + id="unresolved-call", + ), + pytest.param( + [{"role": "user", "content": "first question"}], + [ + { + "type": "function_call", + "call_id": "duplicate", + "name": "lookup", + "arguments": "{}", + }, + { + "type": "custom_tool_call", + "call_id": "duplicate", + "name": "exec", + "input": "pwd", + }, + {"type": "function_call_output", "call_id": "duplicate", "output": "result"}, + ], + id="duplicate-call-id", + ), + pytest.param( + [{"role": "user", "content": "first question"}], + [ + { + "type": "computer_call", + "call_id": "call_computer", + "action": {"type": "screenshot"}, + "status": "completed", + }, + { + "type": "computer_call_output", + "call_id": "call_computer", + "output": {"type": "computer_screenshot", "image_url": "data:image/png;base64,AAAA"}, + }, + ], + id="unsupported-account-scoped-tool", + ), + pytest.param( + [ + { + "type": "custom_tool_call", + "call_id": "call_boundary", + "name": "exec", + "input": "pwd", + } + ], + [{"type": "custom_tool_call_output", "call_id": "call_boundary", "output": "/workspace"}], + id="output-depends-on-prefix-call", + ), + pytest.param( + [ + { + "type": "function_call", + "call_id": "call_reused", + "name": "lookup", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "call_reused", "output": "old"}, + ], + [ + { + "type": "function_call", + "call_id": "call_reused", + "name": "lookup_again", + "arguments": "{}", + }, + {"type": "function_call_output", "call_id": "call_reused", "output": "new"}, + ], + id="call-id-reused-from-prefix", + ), + pytest.param( + [{"role": "user", "content": "first question"}], + [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"role": "user", "content": "next question"}, + ], + id="completed-response-path-is-not-tool-alternative", + ), + pytest.param( + [ + { + "type": "computer_call", + "call_id": "old_computer", + "action": {"type": "screenshot"}, + "status": "completed", + } + ], + [ + { + "type": "custom_tool_call", + "call_id": "call_2", + "name": "exec", + "input": "pwd", + }, + {"type": "custom_tool_call_output", "call_id": "call_2", "output": "/workspace"}, + ], + id="whole-history-is-not-self-contained", + ), + ], +) +def test_quarantine_tool_continuation_rejects_incomplete_or_unsafe_history( + stored_input: list[JsonValue], + suffix: list[JsonValue], +) -> None: + projection = project_responses_input_for_account_neutral_fresh_replay( + [*stored_input, *suffix], + stored_count=len(stored_input), + ) + + assert projection is not None + assert ( + responses_input_has_self_contained_tool_continuation_suffix( + projection.input_items, + stored_count=projection.stored_prefix_count, + ) + is False + ) + + @pytest.mark.parametrize( ("suffix", "pending_tool_calls", "expected"), [