From 117a475aa2b0fd4960a3bc6535173e849f2f71b6 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:44:57 +0200 Subject: [PATCH 1/8] fix(proxy): keep file-pin owner on soft 1011 reconnect A live input_file pin is hard ownership. Soft HTTP-bridge reconnect after upstream 1011 was treating that owner as skippable prompt-cache locality. --- .../proxy/_service/http_bridge/mixin.py | 9 +- .../_service/http_bridge/request_submit.py | 1 + .../.openspec.yaml | 2 + .../context.md | 29 +++ .../design.md | 51 +++++ .../proposal.md | 33 ++++ .../specs/responses-api-compat/spec.md | 32 ++++ .../tasks.md | 21 +++ openspec/specs/responses-api-compat/spec.md | 31 +++ tests/unit/test_proxy_http_bridge.py | 176 ++++++++++++++++++ 10 files changed, 382 insertions(+), 3 deletions(-) create mode 100644 openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/.openspec.yaml create mode 100644 openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/context.md create mode 100644 openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/design.md create mode 100644 openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/proposal.md create mode 100644 openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index c5b8b4d126..a8ec86c23c 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -2044,9 +2044,12 @@ async def _reconnect_http_bridge_session( session.api_key = request_state.api_key forced_refresh_account_id = request_state.force_refresh_account_id excluded_account_ids: set[str] = set(request_state.excluded_account_ids) - requested_preferred_account_id = ( - request_state.preferred_account_id if require_preferred_account or account_neutral_recovery else None - ) + if request_state.file_required_preferred_account: + requested_preferred_account_id = request_state.preferred_account_id or session.account.id + elif require_preferred_account or account_neutral_recovery: + requested_preferred_account_id = request_state.preferred_account_id + else: + requested_preferred_account_id = None required_preferred_account_id = resolve_required_account_id( ("requested reconnect owner", requested_preferred_account_id), ("account-neutral recovery", session.account.id if account_neutral_recovery else None), diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 23216a8c36..ddc6a41dba 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -2826,6 +2826,7 @@ async def _retry_http_bridge_request_on_fresh_upstream( request_state=request_state, restart_reader=True, require_same_account=require_same_account, + require_preferred_account=request_state.file_required_preferred_account, ) if send_request: retry_text_data = self._http_bridge_text_with_account_installation_id( diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/.openspec.yaml b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/context.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/context.md new file mode 100644 index 0000000000..cb7818c2f9 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/context.md @@ -0,0 +1,29 @@ +# Keep file-pin owner on soft 1011 reconnect + +## Purpose + +Close the HTTP-bridge reconnect hole where a live `input_file.file_id` pin is +treated as skippable prompt-cache locality after upstream close `1011`. + +## Decision + +Honor `file_required_preferred_account` in reconnect owner resolution, and +pass it from submit-on-closed fresh-upstream retry. Do not persist pins +across replicas here. + +## Constraints + +File pins are hard ownership. Soft `1011` skip-same-account stays valid only +when no live file pin (and no other required owner) is present. + +## Failure mode + +If the pin account is excluded or cannot reconnect, fail closed with the +existing required-owner unavailable error. Do not fall back to another +account and forward the `file_id`. + +## Example + +Upload `file_xyz` on account A, then send `/v1/responses` with that +`input_file` on a soft prompt-cache bridge session. Upstream closes `1011` +before the next turn is accepted. Reconnect must keep account A required. diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/design.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/design.md new file mode 100644 index 0000000000..8505b71389 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/design.md @@ -0,0 +1,51 @@ +## Context + +`_reconnect_http_bridge_session` promotes `request_state.preferred_account_id` +to a required owner only when the caller sets `require_preferred_account` or +the session is account-neutral. Submit-on-closed recovery calls +`_retry_http_bridge_request_on_fresh_upstream`, which passes +`require_same_account` only for hard keys and never passes +`require_preferred_account`. After upstream `1011`, a soft `prompt_cache` +session therefore sets `skip_same_account`, excludes the file owner, and +allows fallback. The later precreated-recovery path already pins files. + +The existing file-pin requirement already says a live pin MUST override +prompt-cache locality. This change closes the reconnect hole rather than +inventing a new ownership model. + +## Goals / Non-Goals + +**Goals:** + +- Soft `1011` reconnect of a file-pinned request keeps the pin account + required, or fail-closes if that account is excluded or unavailable. +- Movable soft `1011` reconnects without a live file pin still skip the + closed account. + +**Non-Goals:** + +- Durable cross-replica pin persistence (open `#1521`). +- Changing hard-session `1011` keep-owner behavior. +- Changing compact or native WebSocket file routing. + +## Decisions + +- Honor `file_required_preferred_account` inside reconnect owner resolution + so every reconnect caller is covered, not only submit-on-closed. +- Also pass `require_preferred_account` from + `_retry_http_bridge_request_on_fresh_upstream` so that path matches the + already-correct precreated recovery call. +- If the file-required flag is set but `preferred_account_id` is missing, + use the current session account (the session was already on the pin owner). + +**Alternative considered:** only change the one call site. Rejected because +reconnect still ignores `file_required_preferred_account`, so a future +caller can reopen the hole. + +## Risks / Trade-offs + +- [Risk] A file-pinned request can no longer leave a `1011`-closed soft + session's account. → Mitigation: that is the required contract; fail closed + instead of sending the file to another account. +- [Risk] Existing unit tests assert the fresh-upstream retry call shape. + → Mitigation: update the no-file assertion and add a file-pin assertion. diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/proposal.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/proposal.md new file mode 100644 index 0000000000..9cc3266886 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/proposal.md @@ -0,0 +1,33 @@ +## Why + +A live `input_file.file_id` pin is hard ownership and must stay on the +uploading account. Soft HTTP-bridge reconnect after upstream `1011` currently +treats that owner as skippable prompt-cache locality, so submit-on-closed +recovery can send the file to another account. + +## What Changes + +- Treat `file_required_preferred_account` as a required reconnect owner, even + when the session key is soft and the close code is `1011`. +- Pass that requirement from submit-on-closed fresh-upstream retry so it + cannot drop the pin. +- Keep `1011` skip-same-account for movable soft sessions that have no live + file pin. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `responses-api-compat`: HTTP-bridge reconnect after `1011` must keep a live + file-pin owner required, or fail closed. + +## Impact + +- `app/modules/proxy/_service/http_bridge/mixin.py` reconnect owner resolution. +- `app/modules/proxy/_service/http_bridge/request_submit.py` fresh-upstream retry. +- Unit coverage next to the existing hard-`1011` reconnect tests. +- No API, schema, dashboard, or settings changes. diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..2eb1cb3218 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md @@ -0,0 +1,32 @@ +## ADDED Requirements + +### Requirement: Soft HTTP-bridge 1011 reconnect keeps a live file-pin owner + +A still-unsubmitted HTTP-bridge reconnect MUST keep a live `input_file.file_id` +pin as a required owner after a soft session closes with `1011`. +When an HTTP-bridge session is soft (prompt-cache or request locality) and +upstream closed it with `1011`, a still-unsubmitted request that carries a +live `input_file.file_id` pin MUST keep that pin account as a required +reconnect owner. The proxy MUST NOT exclude that account solely because the +close code was `1011`, and MUST NOT fall back to another account while the +pin is live. If the required pin account is already excluded or cannot be +reconnected, the proxy MUST fail closed with the existing required-owner +unavailable error. A soft `1011` reconnect that has no live file pin and no +other required owner MAY still skip the closed account. + +#### Scenario: Soft 1011 reconnect keeps the file-pin account required + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **WHEN** the proxy reconnects that session +- **THEN** account selection MUST treat `account_a` as the required owner +- **AND** it MUST NOT add `account_a` to the excluded-account set solely because of `1011` +- **AND** it MUST NOT enable preferred-account fallback to another account + +#### Scenario: Soft 1011 reconnect without a file pin may skip the closed account + +- **GIVEN** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the still-unsubmitted request has no live file pin and no other required owner +- **WHEN** the proxy reconnects that session +- **THEN** account selection MAY exclude `account_a` and choose another eligible account diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md new file mode 100644 index 0000000000..8b629634a7 --- /dev/null +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md @@ -0,0 +1,21 @@ +## 1. Implementation + +- [x] 1.1 Treat `file_required_preferred_account` as a required owner in + `_reconnect_http_bridge_session`. +- [x] 1.2 Pass `require_preferred_account` from + `_retry_http_bridge_request_on_fresh_upstream` when a live file pin is + present. + +## 2. Regression coverage + +- [x] 2.1 Assert soft `1011` reconnect with a live file pin keeps the owner + required and does not exclude it. +- [x] 2.2 Assert soft `1011` reconnect without a file pin may still skip the + closed account. +- [x] 2.3 Update the fresh-upstream retry call-shape assertion for the new + `require_preferred_account` argument. + +## 3. Validation + +- [x] 3.1 Run the focused HTTP-bridge reconnect unit tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index 450c903864..c5d8fd6e94 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -977,6 +977,37 @@ When multiple `file_id`s are referenced, all live pins MUST resolve to the same - **THEN** the proxy MUST forward the `file_id` verbatim under ordinary unpinned routing - **AND** it MUST NOT reject the request solely because local owner metadata is absent +### Requirement: Soft HTTP-bridge 1011 reconnect keeps a live file-pin owner + +A still-unsubmitted HTTP-bridge reconnect MUST keep a live `input_file.file_id` +pin as a required owner after a soft session closes with `1011`. +When an HTTP-bridge session is soft (prompt-cache or request locality) and +upstream closed it with `1011`, a still-unsubmitted request that carries a +live `input_file.file_id` pin MUST keep that pin account as a required +reconnect owner. The proxy MUST NOT exclude that account solely because the +close code was `1011`, and MUST NOT fall back to another account while the +pin is live. If the required pin account is already excluded or cannot be +reconnected, the proxy MUST fail closed with the existing required-owner +unavailable error. A soft `1011` reconnect that has no live file pin and no +other required owner MAY still skip the closed account. + +#### Scenario: Soft 1011 reconnect keeps the file-pin account required + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **WHEN** the proxy reconnects that session +- **THEN** account selection MUST treat `account_a` as the required owner +- **AND** it MUST NOT add `account_a` to the excluded-account set solely because of `1011` +- **AND** it MUST NOT enable preferred-account fallback to another account + +#### Scenario: Soft 1011 reconnect without a file pin may skip the closed account + +- **GIVEN** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the still-unsubmitted request has no live file pin and no other required owner +- **WHEN** the proxy reconnects that session +- **THEN** account selection MAY exclude `account_a` and choose another eligible account + ### Requirement: Codex backend session_id preserves account affinity When a backend Codex Responses or compact request includes a non-empty accepted session header, the service MUST use that value as the routing affinity key for upstream account selection unless the client supplied a non-empty `x-codex-turn-state` header. If the request lacks a client-supplied `prompt_cache_key`, the service MUST derive and attach a stable `prompt_cache_key` before upstream forwarding so account affinity and upstream prompt-cache routing can coexist. Accepted session headers are `session_id`, `session-id`, `x-codex-session-id`, `x-codex-conversation-id`, and `thread-id`, in that priority order. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 1ede9d331b..72683163aa 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -8677,6 +8677,124 @@ async def ensure_fresh(account: object, **_: object) -> object: assert session.last_upstream_close_code is None +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_keeps_soft_file_pin_owner_after_1011( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011", None), + key_value="sid-soft-file-1011", + ) + session.last_upstream_close_code = 1011 + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + + # when + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert selection_kwargs[0]["preferred_account_id"] == "acc-bridge" + exclude_account_ids = cast(set[str], selection_kwargs[0]["exclude_account_ids"]) + assert "acc-bridge" not in exclude_account_ids + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is False + assert session.account.id == "acc-bridge" + + +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_skips_soft_account_after_1011_without_file_pin( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-1011", None), + key_value="sid-soft-1011", + ) + session.last_upstream_close_code = 1011 + other_account = cast(Any, SimpleNamespace(id="acc-other", status=AccountStatus.ACTIVE, plan_type="plus")) + selection_kwargs: list[dict[str, object]] = [] + + async def select_account(_deadline: float, **kwargs: object) -> proxy_service.AccountSelection: + selection_kwargs.append(kwargs) + return proxy_service.AccountSelection(account=other_account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + upstream = cast(Any, SimpleNamespace(response_header=lambda _name: None, close=AsyncMock())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=False, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", AsyncMock(return_value=upstream)) + + # when + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + exclude_account_ids = cast(set[str], selection_kwargs[0]["exclude_account_ids"]) + assert "acc-bridge" in exclude_account_ids + assert selection_kwargs[0]["preferred_account_id"] is None + assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is True + + @pytest.mark.asyncio async def test_reconnect_http_bridge_session_fails_closed_when_bound_account_is_excluded( monkeypatch: pytest.MonkeyPatch, @@ -20155,10 +20273,68 @@ async def test_retry_http_bridge_request_on_fresh_upstream_reconnects_without_re request_state=request_state, restart_reader=True, require_same_account=False, + require_preferred_account=False, ) send_text.assert_not_awaited() +@pytest.mark.asyncio +async def test_retry_http_bridge_request_on_fresh_upstream_requires_file_pin_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-file", None), + headers={}, + affinity=proxy_service._AffinityPolicy( + key="sid-file", + kind=proxy_service.StickySessionKind.PROMPT_CACHE, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-file", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-file", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + preferred_account_id="acc-file", + file_required_preferred_account=True, + transport="http", + ) + reconnect = AsyncMock() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + # when + recovered = await service._retry_http_bridge_request_on_fresh_upstream( + session=session, + request_state=request_state, + text_data='{"type":"response.create"}', + send_request=False, + ) + + # then + assert recovered is True + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + restart_reader=True, + require_same_account=False, + require_preferred_account=True, + ) + + @pytest.mark.asyncio async def test_retry_http_bridge_request_on_fresh_upstream_refuses_after_response_event( monkeypatch: pytest.MonkeyPatch, From 3a7b04f1e63763b17b31203f704e50af854beab0 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:25:22 +0200 Subject: [PATCH 2/8] fix(proxy): stay under http-bridge mixin line ratchet Move reconnect owner resolution out of mixin.py so the architecture check stays green, and update the remaining fresh-upstream call-shape assertion. --- .../proxy/_service/http_bridge/mixin.py | 14 +++++--------- app/modules/proxy/continuity.py | 19 +++++++++++++++++++ tests/unit/test_proxy_utils.py | 1 + 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index a8ec86c23c..9e08a58f09 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -210,12 +210,11 @@ ) from app.modules.proxy.continuity import ( is_http_bridge_account_neutral_replay, + resolve_reconnect_preferred_account_id, resolve_required_account_id, without_http_bridge_session_affinity_headers, ) -from app.modules.proxy.durable_bridge_coordinator import ( - DurableBridgeLookup, -) +from app.modules.proxy.durable_bridge_coordinator import DurableBridgeLookup from app.modules.proxy.load_balancer import CONTINUITY_OWNER_UNAVAILABLE, AccountLease from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response @@ -2044,12 +2043,9 @@ async def _reconnect_http_bridge_session( session.api_key = request_state.api_key forced_refresh_account_id = request_state.force_refresh_account_id excluded_account_ids: set[str] = set(request_state.excluded_account_ids) - if request_state.file_required_preferred_account: - requested_preferred_account_id = request_state.preferred_account_id or session.account.id - elif require_preferred_account or account_neutral_recovery: - requested_preferred_account_id = request_state.preferred_account_id - else: - requested_preferred_account_id = None + requested_preferred_account_id = resolve_reconnect_preferred_account_id( + request_state, session.account.id, require_preferred_account, account_neutral_recovery + ) required_preferred_account_id = resolve_required_account_id( ("requested reconnect owner", requested_preferred_account_id), ("account-neutral recovery", session.account.id if account_neutral_recovery else None), diff --git a/app/modules/proxy/continuity.py b/app/modules/proxy/continuity.py index c26ba9a93b..614276917a 100644 --- a/app/modules/proxy/continuity.py +++ b/app/modules/proxy/continuity.py @@ -5,6 +5,7 @@ import logging from collections.abc import Mapping from hashlib import sha256 +from typing import Protocol from app.core.clients.proxy import ProxyResponseError from app.core.errors import openai_error @@ -54,6 +55,24 @@ def without_http_bridge_session_affinity_headers(headers: Mapping[str, str]) -> } +class _ReconnectPreferredOwner(Protocol): + preferred_account_id: str | None + file_required_preferred_account: bool + + +def resolve_reconnect_preferred_account_id( + request_state: _ReconnectPreferredOwner, + session_account_id: str, + require_preferred_account: bool, + account_neutral_recovery: bool, +) -> str | None: + if request_state.file_required_preferred_account: + return request_state.preferred_account_id or session_account_id + if require_preferred_account or account_neutral_recovery: + return request_state.preferred_account_id + return None + + def resolve_required_account_id(*owners: tuple[str, str | None]) -> str | None: """Return one proven owner or fail closed when hard sources disagree.""" resolved = [(source, account_id) for source, account_id in owners if account_id is not None] diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 9ed5b4d98f..ba53786c34 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -41959,6 +41959,7 @@ async def capture_send_text(_text: str) -> None: request_state=request_state, restart_reader=True, require_same_account=False, + require_preferred_account=False, ) send_text.assert_awaited_once_with('{"type":"response.create","model":"gpt-5.1","input":"retry"}') assert send_request_ids == ["archive_bridge_retry_fresh"] From 6234fb98fa55138ec7a8a16fcbda3879f26a7976 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:56:27 +0200 Subject: [PATCH 3/8] fix(proxy): surface file-pin owner-unavailable on soft reconnect Fresh-upstream retry swallowed the required-owner envelope and the submit-on-closed path replaced it with generic upstream_unavailable. --- .../_service/http_bridge/request_submit.py | 6 +++ tests/unit/test_proxy_http_bridge.py | 51 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index ddc6a41dba..0e690ccae0 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -78,6 +78,7 @@ _await_task_deferring_cancellation, _build_http_bridge_prewarm_text, _http_bridge_durable_lease_ttl_seconds, + _http_bridge_is_previous_response_owner_unavailable, _http_bridge_key_strength, _http_bridge_precreated_retry_failure_error, _http_bridge_prewarm_enabled, @@ -2847,6 +2848,11 @@ async def _retry_http_bridge_request_on_fresh_upstream( # owner retire the whole session with the typed, non-replayable # failure instead of falling back to the earlier close reason. raise + except ProxyResponseError as exc: + if _http_bridge_is_previous_response_owner_unavailable(exc): + raise + logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True) + return False except Exception: logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True) return False diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 72683163aa..464856229b 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -20335,6 +20335,57 @@ async def test_retry_http_bridge_request_on_fresh_upstream_requires_file_pin_own ) +@pytest.mark.asyncio +async def test_retry_http_bridge_request_on_fresh_upstream_propagates_file_owner_unavailable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-file-unavail", None), + headers={}, + affinity=proxy_service._AffinityPolicy( + key="sid-file-unavail", + kind=proxy_service.StickySessionKind.PROMPT_CACHE, + ), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-file", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=AsyncMock(), close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-file-unavail", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + preferred_account_id="acc-file", + file_required_preferred_account=True, + transport="http", + ) + owner_unavailable = http_bridge_helpers_module._http_bridge_previous_response_owner_unavailable_error() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", AsyncMock(side_effect=owner_unavailable)) + + # when / then + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._retry_http_bridge_request_on_fresh_upstream( + session=session, + request_state=request_state, + text_data='{"type":"response.create"}', + send_request=False, + ) + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + + @pytest.mark.asyncio async def test_retry_http_bridge_request_on_fresh_upstream_refuses_after_response_event( monkeypatch: pytest.MonkeyPatch, From 65a634d8889a397530d502972b488b6a6bd76de8 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:26:51 +0200 Subject: [PATCH 4/8] fix(proxy): map required-owner reconnect miss to owner-unavailable Terminal reconnect selection failures (no_accounts, preferred miss, local cap) were still emitted as generic selection errors even when a live file pin made the owner required. --- .../proxy/_service/http_bridge/helpers.py | 11 ++++ .../proxy/_service/http_bridge/mixin.py | 4 +- .../specs/responses-api-compat/spec.md | 10 +++ .../tasks.md | 2 + openspec/specs/responses-api-compat/spec.md | 10 +++ tests/unit/test_proxy_http_bridge.py | 63 +++++++++++++++++++ 6 files changed, 98 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 27072ceaf1..2ae44e7179 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -189,6 +189,7 @@ RING_STALE_THRESHOLD_SECONDS, RingMembershipService, ) +from app.modules.proxy.selection_errors import selection_failure_response logger = logging.getLogger("app.modules.proxy.service") _TASK_CANCEL_TIMEOUT_SECONDS = 1.0 @@ -2502,6 +2503,16 @@ def _http_bridge_previous_response_owner_unavailable_error() -> ProxyResponseErr ) +def _http_bridge_reconnect_selection_failure( + selection: Any, + required_preferred_account_id: str | None, +) -> ProxyResponseError: + if required_preferred_account_id is not None: + return _http_bridge_previous_response_owner_unavailable_error() + status_code, error_payload = selection_failure_response(selection) + return ProxyResponseError(status_code, error_payload) + + def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyResponseError) -> bool: payload = exc.payload if not isinstance(payload, dict): diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index 9e08a58f09..253e348e8b 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -96,6 +96,7 @@ _http_bridge_parallel_fork_key, _http_bridge_previous_response_alias_key, _http_bridge_previous_response_owner_unavailable_error, + _http_bridge_reconnect_selection_failure, _http_bridge_request_budget_seconds, _http_bridge_request_needs_unanchored_handoff, _http_bridge_session_account_active, @@ -2243,9 +2244,8 @@ def require_bound_account() -> None: preferred_candidate_id = None continue record_selected_account_takeover(None) - status_code, error_payload = selection_failure_response(selection) complete_failed_handoff() - raise ProxyResponseError(status_code, error_payload) + raise _http_bridge_reconnect_selection_failure(selection, required_preferred_account_id) if required_preferred_account_id is not None and account.id != required_preferred_account_id: if selection.lease is not None: selected_account_lease = selection.lease diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md index 2eb1cb3218..ecf325c2ed 100644 --- a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md @@ -30,3 +30,13 @@ other required owner MAY still skip the closed account. - **AND** the still-unsubmitted request has no live file pin and no other required owner - **WHEN** the proxy reconnects that session - **THEN** account selection MAY exclude `account_a` and choose another eligible account + +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be selected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection cannot return `account_a` +- **WHEN** the proxy reconnects that session +- **THEN** the proxy MUST fail closed with the existing required-owner unavailable error +- **AND** it MUST NOT replace that envelope with a generic selection failure diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md index 8b629634a7..bce827dc21 100644 --- a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md @@ -14,6 +14,8 @@ closed account. - [x] 2.3 Update the fresh-upstream retry call-shape assertion for the new `require_preferred_account` argument. +- [x] 2.4 Assert soft `1011` file-pin reconnect fails closed with the + required-owner envelope when selection cannot return the pin account. ## 3. Validation diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index c5d8fd6e94..75a39a2d82 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -1008,6 +1008,16 @@ other required owner MAY still skip the closed account. - **WHEN** the proxy reconnects that session - **THEN** account selection MAY exclude `account_a` and choose another eligible account +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be selected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection cannot return `account_a` +- **WHEN** the proxy reconnects that session +- **THEN** the proxy MUST fail closed with the existing required-owner unavailable error +- **AND** it MUST NOT replace that envelope with a generic selection failure + ### Requirement: Codex backend session_id preserves account affinity When a backend Codex Responses or compact request includes a non-empty accepted session header, the service MUST use that value as the routing affinity key for upstream account selection unless the client supplied a non-empty `x-codex-turn-state` header. If the request lacks a client-supplied `prompt_cache_key`, the service MUST derive and attach a stable `prompt_cache_key` before upstream forwarding so account affinity and upstream prompt-cache routing can coexist. Accepted session headers are `session_id`, `session-id`, `x-codex-session-id`, `x-codex-conversation-id`, and `thread-id`, in that priority order. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 464856229b..11af4a8e27 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -8795,6 +8795,69 @@ async def ensure_fresh(account: object, **_: object) -> object: assert selection_kwargs[0]["fallback_on_preferred_account_unavailable"] is True +@pytest.mark.asyncio +@pytest.mark.parametrize( + "selection_error_code", + ["no_accounts", "preferred_account_unavailable", "account_stream_cap"], +) +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_cannot_be_selected( + monkeypatch: pytest.MonkeyPatch, + selection_error_code: str, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-miss", None), + key_value="sid-soft-file-1011-miss", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection( + account=None, + error_message="No available accounts", + error_code=selection_error_code, + ) + + async def sleep_for_recovery(*_args: object, **_kwargs: object) -> bool: + return False + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-miss", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(http_bridge_mixin_module, "_sleep_for_account_selection_recovery", sleep_for_recovery) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_reconnect_http_bridge_session_fails_closed_when_bound_account_is_excluded( monkeypatch: pytest.MonkeyPatch, From 6230c97b9d11ac42803b433db2bc04d89391955d Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:36:11 +0200 Subject: [PATCH 5/8] test(proxy): expect owner-unavailable on file-owner reconnect miss Precreated retry already fail-closes when the required file owner cannot be selected; the envelope is now previous_response_owner_unavailable. --- tests/unit/test_proxy_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index ba53786c34..36130ee2a3 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -43178,7 +43178,7 @@ async def select_account(_deadline: float, **kwargs: object) -> AccountSelection assert request_state.previous_response_id == "resp_file_anchor" assert request_state.preferred_account_id == owner_account.id assert request_state.excluded_account_ids == set() - assert request_state.error_code_override == "no_accounts" + assert request_state.error_code_override == "previous_response_owner_unavailable" @pytest.mark.asyncio From 9eaac241bd3a823cc7c957c78c58e40ebae41163 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:56:42 +0200 Subject: [PATCH 6/8] fix(proxy): map required-owner connect miss to owner-unavailable Selecting the file-pin owner and then failing to open its replacement socket still collapsed submit-on-closed into generic upstream_unavailable. --- .../proxy/_service/http_bridge/helpers.py | 9 ++ .../proxy/_service/http_bridge/mixin.py | 5 +- .../specs/responses-api-compat/spec.md | 11 ++ .../tasks.md | 2 + openspec/specs/responses-api-compat/spec.md | 11 ++ tests/unit/test_proxy_http_bridge.py | 134 ++++++++++++++++++ 6 files changed, 170 insertions(+), 2 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 2ae44e7179..873927d884 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -2513,6 +2513,15 @@ def _http_bridge_reconnect_selection_failure( return ProxyResponseError(status_code, error_payload) +def _http_bridge_reconnect_connect_failure( + exc: ProxyResponseError, + required_preferred_account_id: str | None, +) -> ProxyResponseError: + if required_preferred_account_id is not None: + return _http_bridge_previous_response_owner_unavailable_error() + return exc + + def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyResponseError) -> bool: payload = exc.payload if not isinstance(payload, dict): diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index 253e348e8b..7f89ee2a29 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -96,6 +96,7 @@ _http_bridge_parallel_fork_key, _http_bridge_previous_response_alias_key, _http_bridge_previous_response_owner_unavailable_error, + _http_bridge_reconnect_connect_failure, _http_bridge_reconnect_selection_failure, _http_bridge_request_budget_seconds, _http_bridge_request_needs_unanchored_handoff, @@ -2290,7 +2291,7 @@ def require_bound_account() -> None: if exc.status_code != 401 or _remaining_budget_seconds(deadline) <= 0: await release_selected_account_lease() complete_failed_handoff() - raise + raise _http_bridge_reconnect_connect_failure(exc, required_preferred_account_id) from exc try: account = await self._ensure_fresh_with_budget( account, @@ -2313,7 +2314,7 @@ def require_bound_account() -> None: if retry_exc.status_code != 401: await release_selected_account_lease() complete_failed_handoff() - raise + raise _http_bridge_reconnect_connect_failure(retry_exc, required_preferred_account_id) await self._handle_proxy_error(account, retry_exc) await abandon_selected_account_retry(account) continue diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md index ecf325c2ed..e8084a6eae 100644 --- a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/specs/responses-api-compat/spec.md @@ -40,3 +40,14 @@ other required owner MAY still skip the closed account. - **WHEN** the proxy reconnects that session - **THEN** the proxy MUST fail closed with the existing required-owner unavailable error - **AND** it MUST NOT replace that envelope with a generic selection failure + +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be connected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection returns `account_a` +- **AND** opening a replacement upstream for `account_a` fails +- **WHEN** the proxy reconnects that session on submit +- **THEN** the client-visible error MUST be the existing required-owner unavailable error +- **AND** it MUST NOT be replaced with a generic `upstream_unavailable` envelope diff --git a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md index bce827dc21..b6cb2a89d2 100644 --- a/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md +++ b/openspec/changes/keep-file-pin-owner-on-soft-1011-reconnect/tasks.md @@ -16,6 +16,8 @@ `require_preferred_account` argument. - [x] 2.4 Assert soft `1011` file-pin reconnect fails closed with the required-owner envelope when selection cannot return the pin account. +- [x] 2.5 Assert submit-on-closed emits the required-owner envelope when + the pin account is selected but the replacement socket cannot be opened. ## 3. Validation diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index 75a39a2d82..e74dd5bbeb 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -1018,6 +1018,17 @@ other required owner MAY still skip the closed account. - **THEN** the proxy MUST fail closed with the existing required-owner unavailable error - **AND** it MUST NOT replace that envelope with a generic selection failure +#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be connected + +- **GIVEN** a live in-memory pin `file_xyz -> account_a` +- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011` +- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz` +- **AND** account selection returns `account_a` +- **AND** opening a replacement upstream for `account_a` fails +- **WHEN** the proxy reconnects that session on submit +- **THEN** the client-visible error MUST be the existing required-owner unavailable error +- **AND** it MUST NOT be replaced with a generic `upstream_unavailable` envelope + ### Requirement: Codex backend session_id preserves account affinity When a backend Codex Responses or compact request includes a non-empty accepted session header, the service MUST use that value as the routing affinity key for upstream account selection unless the client supplied a non-empty `x-codex-turn-state` header. If the request lacks a client-supplied `prompt_cache_key`, the service MUST derive and attach a stable `prompt_cache_key` before upstream forwarding so account affinity and upstream prompt-cache routing can coexist. Accepted session headers are `session_id`, `session-id`, `x-codex-session-id`, `x-codex-conversation-id`, and `thread-id`, in that priority order. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 11af4a8e27..90b1736d18 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -8858,6 +8858,67 @@ async def sleep_for_recovery(*_args: object, **_kwargs: object) -> bool: assert exc_info.value.payload["error"]["type"] == "server_error" +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_connect_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-connect", None), + key_value="sid-soft-file-1011-connect", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(*_args: object, **_kwargs: object) -> Any: + raise proxy_service.ProxyResponseError( + 503, + proxy_service.openai_error("upstream_proxy_unavailable", "Upstream proxy unavailable"), + ) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-connect", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_reconnect_http_bridge_session_fails_closed_when_bound_account_is_excluded( monkeypatch: pytest.MonkeyPatch, @@ -20449,6 +20510,79 @@ async def test_retry_http_bridge_request_on_fresh_upstream_propagates_file_owner assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" +@pytest.mark.asyncio +async def test_submit_http_bridge_request_emits_owner_unavailable_when_file_pin_reconnect_connect_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-submit-file-1011", None), + key_value="sid-submit-file-1011", + ) + session.closed = True + session.last_upstream_close_code = 1011 + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-file-1011", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.4","input":"file"}', + transport="http", + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + skip_request_log=True, + ) + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(*_args: object, **_kwargs: object) -> Any: + raise proxy_service.ProxyResponseError( + 503, + proxy_service.openai_error("upstream_proxy_unavailable", "Upstream proxy unavailable"), + ) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + service._http_bridge_sessions[session.key] = session + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_retry_http_bridge_request_on_fresh_upstream_refuses_after_response_event( monkeypatch: pytest.MonkeyPatch, From bca9ba7f2230c6a2abcf9a5af86b1011fc540261 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:06:50 +0200 Subject: [PATCH 7/8] fix(proxy): map required-owner transport miss to owner-unavailable A budget-expired ClientError or TimeoutError while opening the file-pin owner's replacement socket still collapsed submit-on-closed to a generic upstream_unavailable envelope. --- .../proxy/_service/http_bridge/helpers.py | 6 +- .../proxy/_service/http_bridge/mixin.py | 4 +- tests/unit/test_proxy_http_bridge.py | 65 +++++++++++++++++++ 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 873927d884..ee8f802370 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -2514,12 +2514,14 @@ def _http_bridge_reconnect_selection_failure( def _http_bridge_reconnect_connect_failure( - exc: ProxyResponseError, + exc: BaseException, required_preferred_account_id: str | None, ) -> ProxyResponseError: if required_preferred_account_id is not None: return _http_bridge_previous_response_owner_unavailable_error() - return exc + if isinstance(exc, ProxyResponseError): + return exc + raise exc def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyResponseError) -> bool: diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index 7f89ee2a29..b6172fbea3 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -2336,7 +2336,7 @@ def require_bound_account() -> None: await release_selected_account_lease() complete_failed_handoff() raise - except (aiohttp.ClientError, asyncio.TimeoutError): + except (aiohttp.ClientError, asyncio.TimeoutError) as transport_exc: if selected_is_preferred and _remaining_budget_seconds(deadline) > 0: if retry_same_account_once: retry_same_account_once = False @@ -2346,7 +2346,7 @@ def require_bound_account() -> None: continue await release_selected_account_lease() complete_failed_handoff() - raise + raise _http_bridge_reconnect_connect_failure(transport_exc, required_preferred_account_id) except asyncio.CancelledError: session.closed = True await release_selected_account_lease() diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 90b1736d18..8a4af39a8f 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -8919,6 +8919,71 @@ async def open_upstream(*_args: object, **_kwargs: object) -> Any: assert exc_info.value.payload["error"]["type"] == "server_error" +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_transport_times_out( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-timeout", None), + key_value="sid-soft-file-1011-timeout", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(account: object, **_: object) -> object: + return account + + async def open_upstream(*_args: object, **_kwargs: object) -> Any: + raise aiohttp.ClientError("replacement socket timed out") + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-timeout", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic() - 1.0, + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=0.001, + http_responses_session_bridge_request_budget_seconds=0.001, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + monkeypatch.setattr(service, "_open_upstream_websocket_with_budget", open_upstream) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_reconnect_http_bridge_session_fails_closed_when_bound_account_is_excluded( monkeypatch: pytest.MonkeyPatch, From 9e85128add4db15fdacf87074616278511196e2e Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:15:44 +0200 Subject: [PATCH 8/8] fix(proxy): map required-owner refresh miss to owner-unavailable A budget-expired RefreshError for the file-pin owner still collapsed submit-on-closed into generic upstream_unavailable. --- .../proxy/_service/http_bridge/mixin.py | 2 +- tests/unit/test_proxy_http_bridge.py | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index b6172fbea3..5432e7cf73 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -2335,7 +2335,7 @@ def require_bound_account() -> None: continue await release_selected_account_lease() complete_failed_handoff() - raise + raise _http_bridge_reconnect_connect_failure(exc, required_preferred_account_id) except (aiohttp.ClientError, asyncio.TimeoutError) as transport_exc: if selected_is_preferred and _remaining_budget_seconds(deadline) > 0: if retry_same_account_once: diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 8a4af39a8f..e6f45a8ee1 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -8984,6 +8984,68 @@ async def open_upstream(*_args: object, **_kwargs: object) -> Any: assert exc_info.value.payload["error"]["type"] == "server_error" +@pytest.mark.asyncio +async def test_reconnect_http_bridge_session_fails_closed_when_file_pin_owner_refresh_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "sid-soft-file-1011-refresh", None), + key_value="sid-soft-file-1011-refresh", + ) + session.last_upstream_close_code = 1011 + + async def select_account(_deadline: float, **_kwargs: object) -> proxy_service.AccountSelection: + return proxy_service.AccountSelection(account=session.account, error_message=None, error_code=None) + + async def ensure_fresh(*_args: object, **_kwargs: object) -> Any: + raise RefreshError("invalid_grant", "refresh failed", True) + + request_state = proxy_service._WebSocketRequestState( + request_id="req-soft-file-1011-refresh", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic() - 1.0, + preferred_account_id="acc-bridge", + file_required_preferred_account=True, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + proxy_request_budget_seconds=0.001, + http_responses_session_bridge_request_budget_seconds=0.001, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + prefer_earlier_reset_accounts=False, + routing_strategy=None, + ) + ) + ), + ) + monkeypatch.setattr(service, "_select_account_with_budget_for_stream", select_account) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", ensure_fresh) + service._load_balancer = cast(Any, SimpleNamespace(mark_permanent_failure=AsyncMock())) + + # when + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session(session, request_state=request_state) + + # then + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert exc_info.value.payload["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_reconnect_http_bridge_session_fails_closed_when_bound_account_is_excluded( monkeypatch: pytest.MonkeyPatch,