Skip to content
22 changes: 22 additions & 0 deletions app/modules/proxy/_service/http_bridge/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@
RING_STALE_THRESHOLD_SECONDS,
RingMembershipService,
)
from app.modules.proxy.selection_errors import selection_failure_response

logger = logging.getLogger("app.modules.proxy.service")
_TASK_CANCEL_TIMEOUT_SECONDS = 1.0
Expand Down Expand Up @@ -2502,6 +2503,27 @@ def _http_bridge_previous_response_owner_unavailable_error() -> ProxyResponseErr
)


def _http_bridge_reconnect_selection_failure(
selection: Any,
required_preferred_account_id: str | None,
) -> ProxyResponseError:
if required_preferred_account_id is not None:
return _http_bridge_previous_response_owner_unavailable_error()
status_code, error_payload = selection_failure_response(selection)
return ProxyResponseError(status_code, error_payload)


def _http_bridge_reconnect_connect_failure(
exc: BaseException,
required_preferred_account_id: str | None,
) -> ProxyResponseError:
if required_preferred_account_id is not None:
return _http_bridge_previous_response_owner_unavailable_error()
if isinstance(exc, ProxyResponseError):
return exc
raise exc


def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyResponseError) -> bool:
payload = exc.payload
if not isinstance(payload, dict):
Expand Down
24 changes: 12 additions & 12 deletions app/modules/proxy/_service/http_bridge/mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@
_http_bridge_parallel_fork_key,
_http_bridge_previous_response_alias_key,
_http_bridge_previous_response_owner_unavailable_error,
_http_bridge_reconnect_connect_failure,
_http_bridge_reconnect_selection_failure,
_http_bridge_request_budget_seconds,
_http_bridge_request_needs_unanchored_handoff,
_http_bridge_session_account_active,
Expand Down Expand Up @@ -210,12 +212,11 @@
)
from app.modules.proxy.continuity import (
is_http_bridge_account_neutral_replay,
resolve_reconnect_preferred_account_id,
resolve_required_account_id,
without_http_bridge_session_affinity_headers,
)
from app.modules.proxy.durable_bridge_coordinator import (
DurableBridgeLookup,
)
from app.modules.proxy.durable_bridge_coordinator import DurableBridgeLookup
from app.modules.proxy.load_balancer import CONTINUITY_OWNER_UNAVAILABLE, AccountLease
from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response

Expand Down Expand Up @@ -2044,8 +2045,8 @@ async def _reconnect_http_bridge_session(
session.api_key = request_state.api_key
forced_refresh_account_id = request_state.force_refresh_account_id
excluded_account_ids: set[str] = set(request_state.excluded_account_ids)
requested_preferred_account_id = (
request_state.preferred_account_id if require_preferred_account or account_neutral_recovery else None
requested_preferred_account_id = resolve_reconnect_preferred_account_id(
request_state, session.account.id, require_preferred_account, account_neutral_recovery
)
required_preferred_account_id = resolve_required_account_id(
("requested reconnect owner", requested_preferred_account_id),
Expand Down Expand Up @@ -2244,9 +2245,8 @@ def require_bound_account() -> None:
preferred_candidate_id = None
continue
record_selected_account_takeover(None)
status_code, error_payload = selection_failure_response(selection)
complete_failed_handoff()
raise ProxyResponseError(status_code, error_payload)
raise _http_bridge_reconnect_selection_failure(selection, required_preferred_account_id)
if required_preferred_account_id is not None and account.id != required_preferred_account_id:
if selection.lease is not None:
selected_account_lease = selection.lease
Expand Down Expand Up @@ -2291,7 +2291,7 @@ def require_bound_account() -> None:
if exc.status_code != 401 or _remaining_budget_seconds(deadline) <= 0:
await release_selected_account_lease()
complete_failed_handoff()
raise
raise _http_bridge_reconnect_connect_failure(exc, required_preferred_account_id) from exc
Comment thread
mastertyko marked this conversation as resolved.
Comment thread
mastertyko marked this conversation as resolved.
try:
account = await self._ensure_fresh_with_budget(
account,
Expand All @@ -2314,7 +2314,7 @@ def require_bound_account() -> None:
if retry_exc.status_code != 401:
await release_selected_account_lease()
complete_failed_handoff()
raise
raise _http_bridge_reconnect_connect_failure(retry_exc, required_preferred_account_id)
await self._handle_proxy_error(account, retry_exc)
await abandon_selected_account_retry(account)
continue
Expand All @@ -2335,8 +2335,8 @@ def require_bound_account() -> None:
continue
await release_selected_account_lease()
complete_failed_handoff()
raise
except (aiohttp.ClientError, asyncio.TimeoutError):
raise _http_bridge_reconnect_connect_failure(exc, required_preferred_account_id)
except (aiohttp.ClientError, asyncio.TimeoutError) as transport_exc:
if selected_is_preferred and _remaining_budget_seconds(deadline) > 0:
if retry_same_account_once:
retry_same_account_once = False
Expand All @@ -2346,7 +2346,7 @@ def require_bound_account() -> None:
continue
await release_selected_account_lease()
complete_failed_handoff()
raise
raise _http_bridge_reconnect_connect_failure(transport_exc, required_preferred_account_id)
except asyncio.CancelledError:
session.closed = True
await release_selected_account_lease()
Expand Down
7 changes: 7 additions & 0 deletions app/modules/proxy/_service/http_bridge/request_submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
_await_task_deferring_cancellation,
_build_http_bridge_prewarm_text,
_http_bridge_durable_lease_ttl_seconds,
_http_bridge_is_previous_response_owner_unavailable,
_http_bridge_key_strength,
_http_bridge_precreated_retry_failure_error,
_http_bridge_prewarm_enabled,
Expand Down Expand Up @@ -2826,6 +2827,7 @@ async def _retry_http_bridge_request_on_fresh_upstream(
request_state=request_state,
restart_reader=True,
require_same_account=require_same_account,
require_preferred_account=request_state.file_required_preferred_account,
Comment thread
mastertyko marked this conversation as resolved.
)
if send_request:
retry_text_data = self._http_bridge_text_with_account_installation_id(
Expand All @@ -2846,6 +2848,11 @@ async def _retry_http_bridge_request_on_fresh_upstream(
# owner retire the whole session with the typed, non-replayable
# failure instead of falling back to the earlier close reason.
raise
except ProxyResponseError as exc:
if _http_bridge_is_previous_response_owner_unavailable(exc):
raise
Comment thread
mastertyko marked this conversation as resolved.
Comment thread
mastertyko marked this conversation as resolved.
logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True)
return False
except Exception:
logger.warning("HTTP bridge retry on fresh upstream failed", exc_info=True)
return False
Expand Down
19 changes: 19 additions & 0 deletions app/modules/proxy/continuity.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import logging
from collections.abc import Mapping
from hashlib import sha256
from typing import Protocol

from app.core.clients.proxy import ProxyResponseError
from app.core.errors import openai_error
Expand Down Expand Up @@ -54,6 +55,24 @@ def without_http_bridge_session_affinity_headers(headers: Mapping[str, str]) ->
}


class _ReconnectPreferredOwner(Protocol):
preferred_account_id: str | None
file_required_preferred_account: bool


def resolve_reconnect_preferred_account_id(
request_state: _ReconnectPreferredOwner,
session_account_id: str,
require_preferred_account: bool,
account_neutral_recovery: bool,
) -> str | None:
if request_state.file_required_preferred_account:
return request_state.preferred_account_id or session_account_id
if require_preferred_account or account_neutral_recovery:
return request_state.preferred_account_id
return None


def resolve_required_account_id(*owners: tuple[str, str | None]) -> str | None:
"""Return one proven owner or fail closed when hard sources disagree."""
resolved = [(source, account_id) for source, account_id in owners if account_id is not None]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
schema: spec-driven
created: 2026-08-14
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Keep file-pin owner on soft 1011 reconnect

## Purpose

Close the HTTP-bridge reconnect hole where a live `input_file.file_id` pin is
treated as skippable prompt-cache locality after upstream close `1011`.

## Decision

Honor `file_required_preferred_account` in reconnect owner resolution, and
pass it from submit-on-closed fresh-upstream retry. Do not persist pins
across replicas here.

## Constraints

File pins are hard ownership. Soft `1011` skip-same-account stays valid only
when no live file pin (and no other required owner) is present.

## Failure mode

If the pin account is excluded or cannot reconnect, fail closed with the
existing required-owner unavailable error. Do not fall back to another
account and forward the `file_id`.

## Example

Upload `file_xyz` on account A, then send `/v1/responses` with that
`input_file` on a soft prompt-cache bridge session. Upstream closes `1011`
before the next turn is accepted. Reconnect must keep account A required.
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
## Context

`_reconnect_http_bridge_session` promotes `request_state.preferred_account_id`
to a required owner only when the caller sets `require_preferred_account` or
the session is account-neutral. Submit-on-closed recovery calls
`_retry_http_bridge_request_on_fresh_upstream`, which passes
`require_same_account` only for hard keys and never passes
`require_preferred_account`. After upstream `1011`, a soft `prompt_cache`
session therefore sets `skip_same_account`, excludes the file owner, and
allows fallback. The later precreated-recovery path already pins files.

The existing file-pin requirement already says a live pin MUST override
prompt-cache locality. This change closes the reconnect hole rather than
inventing a new ownership model.

## Goals / Non-Goals

**Goals:**

- Soft `1011` reconnect of a file-pinned request keeps the pin account
required, or fail-closes if that account is excluded or unavailable.
- Movable soft `1011` reconnects without a live file pin still skip the
closed account.

**Non-Goals:**

- Durable cross-replica pin persistence (open `#1521`).
- Changing hard-session `1011` keep-owner behavior.
- Changing compact or native WebSocket file routing.

## Decisions

- Honor `file_required_preferred_account` inside reconnect owner resolution
so every reconnect caller is covered, not only submit-on-closed.
- Also pass `require_preferred_account` from
`_retry_http_bridge_request_on_fresh_upstream` so that path matches the
already-correct precreated recovery call.
- If the file-required flag is set but `preferred_account_id` is missing,
use the current session account (the session was already on the pin owner).

**Alternative considered:** only change the one call site. Rejected because
reconnect still ignores `file_required_preferred_account`, so a future
caller can reopen the hole.

## Risks / Trade-offs

- [Risk] A file-pinned request can no longer leave a `1011`-closed soft
session's account. → Mitigation: that is the required contract; fail closed
instead of sending the file to another account.
- [Risk] Existing unit tests assert the fresh-upstream retry call shape.
→ Mitigation: update the no-file assertion and add a file-pin assertion.
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
## Why

A live `input_file.file_id` pin is hard ownership and must stay on the
uploading account. Soft HTTP-bridge reconnect after upstream `1011` currently
treats that owner as skippable prompt-cache locality, so submit-on-closed
recovery can send the file to another account.

## What Changes

- Treat `file_required_preferred_account` as a required reconnect owner, even
when the session key is soft and the close code is `1011`.
- Pass that requirement from submit-on-closed fresh-upstream retry so it
cannot drop the pin.
- Keep `1011` skip-same-account for movable soft sessions that have no live
file pin.

## Capabilities

### New Capabilities

- None.

### Modified Capabilities

- `responses-api-compat`: HTTP-bridge reconnect after `1011` must keep a live
file-pin owner required, or fail closed.

## Impact

- `app/modules/proxy/_service/http_bridge/mixin.py` reconnect owner resolution.
- `app/modules/proxy/_service/http_bridge/request_submit.py` fresh-upstream retry.
- Unit coverage next to the existing hard-`1011` reconnect tests.
- No API, schema, dashboard, or settings changes.
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
## ADDED Requirements

### Requirement: Soft HTTP-bridge 1011 reconnect keeps a live file-pin owner

A still-unsubmitted HTTP-bridge reconnect MUST keep a live `input_file.file_id`
pin as a required owner after a soft session closes with `1011`.
When an HTTP-bridge session is soft (prompt-cache or request locality) and
upstream closed it with `1011`, a still-unsubmitted request that carries a
live `input_file.file_id` pin MUST keep that pin account as a required
reconnect owner. The proxy MUST NOT exclude that account solely because the
close code was `1011`, and MUST NOT fall back to another account while the
pin is live. If the required pin account is already excluded or cannot be
reconnected, the proxy MUST fail closed with the existing required-owner
unavailable error. A soft `1011` reconnect that has no live file pin and no
other required owner MAY still skip the closed account.

#### Scenario: Soft 1011 reconnect keeps the file-pin account required

- **GIVEN** a live in-memory pin `file_xyz -> account_a`
- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011`
- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz`
- **WHEN** the proxy reconnects that session
- **THEN** account selection MUST treat `account_a` as the required owner
- **AND** it MUST NOT add `account_a` to the excluded-account set solely because of `1011`
- **AND** it MUST NOT enable preferred-account fallback to another account

#### Scenario: Soft 1011 reconnect without a file pin may skip the closed account

- **GIVEN** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011`
- **AND** the still-unsubmitted request has no live file pin and no other required owner
- **WHEN** the proxy reconnects that session
- **THEN** account selection MAY exclude `account_a` and choose another eligible account

#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be selected

- **GIVEN** a live in-memory pin `file_xyz -> account_a`
- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011`
- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz`
- **AND** account selection cannot return `account_a`
- **WHEN** the proxy reconnects that session
- **THEN** the proxy MUST fail closed with the existing required-owner unavailable error
- **AND** it MUST NOT replace that envelope with a generic selection failure

#### Scenario: Soft 1011 file-pin reconnect fails closed when the required owner cannot be connected

- **GIVEN** a live in-memory pin `file_xyz -> account_a`
- **AND** a soft prompt-cache HTTP-bridge session on `account_a` closed with `1011`
- **AND** the next still-unsubmitted `/v1/responses` request references `file_xyz`
- **AND** account selection returns `account_a`
- **AND** opening a replacement upstream for `account_a` fails
- **WHEN** the proxy reconnects that session on submit
- **THEN** the client-visible error MUST be the existing required-owner unavailable error
- **AND** it MUST NOT be replaced with a generic `upstream_unavailable` envelope
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
## 1. Implementation

- [x] 1.1 Treat `file_required_preferred_account` as a required owner in
`_reconnect_http_bridge_session`.
- [x] 1.2 Pass `require_preferred_account` from
`_retry_http_bridge_request_on_fresh_upstream` when a live file pin is
present.

## 2. Regression coverage

- [x] 2.1 Assert soft `1011` reconnect with a live file pin keeps the owner
required and does not exclude it.
- [x] 2.2 Assert soft `1011` reconnect without a file pin may still skip the
closed account.
- [x] 2.3 Update the fresh-upstream retry call-shape assertion for the new
`require_preferred_account` argument.
- [x] 2.4 Assert soft `1011` file-pin reconnect fails closed with the
required-owner envelope when selection cannot return the pin account.
- [x] 2.5 Assert submit-on-closed emits the required-owner envelope when
the pin account is selected but the replacement socket cannot be opened.

## 3. Validation

- [x] 3.1 Run the focused HTTP-bridge reconnect unit tests.
- [x] 3.2 Run strict OpenSpec validation for this change.
Loading
Loading