diff --git a/.env.example b/.env.example index bc4da900..fb0cc5f1 100644 --- a/.env.example +++ b/.env.example @@ -78,6 +78,28 @@ RECON_JOB_TTL_S=86400 # Disk tile cache root — safe to delete anytime, refills on demand. TILE_CACHE_DIR=./data/tilecache +# ── Outbound proxy pool (optional; empty = off) ────────────────────────────── +# Comma-separated proxies the shared upstream client rotates over. Useful +# because several upstreams gate on the CALLER'S IP rather than a key: +# airplanes.live 403s a datacenter address, adsb.lol rate-limits per source IP, +# and OpenSky's anonymous credit budget is per IP. +# UPSTREAM_PROXIES=http://user:pass@host:8080,socks5://host:1080 +# Point this ONLY at egress you own or pay for. There is deliberately no +# discovery and no bundled list: public "free proxy" pools are largely consumer +# machines enrolled by malware whose owners never agreed to carry traffic, plus +# interception nodes that would read every query this console makes. +# An explicit pool here outranks HTTPS_PROXY; NO_PROXY exclusions still apply, +# and the localhost sidecars (:8090, :8093) always bypass the pool. +UPSTREAM_PROXIES= +# A proxy that errors is skipped for this long before being retried. +UPSTREAM_PROXY_COOLDOWN_S=300 +# How many pool members one request may try before giving up. +UPSTREAM_PROXY_MAX_TRIES=3 +# When every proxy is cooling down: 0 (default) fails the request, 1 sends it +# directly. Default 0 on purpose — configuring a proxy usually means "not from +# my own address", and a silent direct fallback would leak exactly that. +UPSTREAM_PROXY_FALLBACK_DIRECT= + # ── MCP server + local AI (Ollama) ─────────────────────────────────────────── API_BASE=http://localhost:8000 OLLAMA_HOST=http://localhost:11434 diff --git a/CLAUDE.md b/CLAUDE.md index fadd5595..7d341ea7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -155,9 +155,9 @@ Ontology (2026-07-07, docs/decisions.md#ontology-local-first-store-2026-07-07): - Backend tests from the **repo ROOT** (from `apps/api` the `.env` auth resolves → wall of 401s): `OSINT_DISABLE_BACKGROUND=1 apps/api/.venv/bin/pytest apps/api -q` - Baseline: **2108 passed + 2 skipped in ~110 s** (skip = opt-in live probes; - measured 2026-07-30, branch overnight-provenance-answers-2026-07-29 — see - `docs/exec-report-2026-07-29.md`). Runs SERIAL by default: `-n auto --dist + Baseline: **2141 passed + 2 skipped in ~333 s** (skip = opt-in live probes; + measured 2026-07-31, branch repo-setup-ui-access, upstream proxy pool wave). + Runs SERIAL by default: `-n auto --dist loadfile` groups different files per worker on different core counts, so a suite with module-state leaks answers differently per machine and CI (4 cores) failed a branch that was green locally (16). Opt in per-machine, never commit diff --git a/apps/api/app/config.py b/apps/api/app/config.py index 74855429..956871b6 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -127,6 +127,28 @@ class Settings(BaseSettings): # union for deploys without a sidecar; the local .env sets ADSB_SIDECAR_ONLY=1. adsb_sidecar_only: bool = False + # ── outbound proxy pool (app.upstream_proxy) ── + # Comma-separated proxy URLs the shared upstream client rotates over, e.g. + # "http://user:pass@host:8080,socks5://host:1080". EMPTY = off, which is the + # default and leaves the client wired exactly as it was. + # Point this ONLY at egress you own or pay for. There is no discovery and no + # bundled list: public "free proxy" pools are mostly malware-enrolled home + # machines and interception points, so they would route this console's + # queries through devices whose owners never agreed to carry them. + # Useful because several upstreams gate on the caller's IP rather than a + # key — airplanes.live 403s a datacenter address, adsb.lol rate-limits per + # source IP, OpenSky's anonymous credit budget is per IP. + upstream_proxies: str = "" # UPSTREAM_PROXIES + # A proxy that errors is skipped for this long before being retried. + upstream_proxy_cooldown_s: float = 300.0 + # How many pool members one request may try before giving up. + upstream_proxy_max_tries: int = 3 + # When every proxy is cooling down: False (default) fails the request, True + # sends it directly. Default False on purpose — configuring a proxy usually + # means "not from my own address", and a silent direct fallback would leak + # exactly what the proxy was there to avoid. + upstream_proxy_fallback_direct: bool = False + # ── infra ── database_url: str = "postgresql+asyncpg://osint:osint@localhost:5432/osint" redis_url: str = "redis://localhost:6379/0" diff --git a/apps/api/app/upstream.py b/apps/api/app/upstream.py index 83a73492..1d491dff 100644 --- a/apps/api/app/upstream.py +++ b/apps/api/app/upstream.py @@ -18,23 +18,106 @@ _CLIENT: httpx.AsyncClient | None = None +def _transport(proxy: str | None = None) -> httpx.AsyncHTTPTransport: + """One transport shape, optionally routed through `proxy`. + + local_address pins outbound sockets to IPv4. Several upstreams + (CloudFront-backed weathercam.digitraffic.fi, cwwp2.dot.ca.gov) publish + AAAA records; on hosts with broken IPv6 egress httpx exhausts the v6 + attempts and reports "All connection attempts failed" while curl quietly + falls back. One retry absorbs transient resets on long-lived pooled + connections. + """ + return httpx.AsyncHTTPTransport( + local_address="0.0.0.0", + retries=1, + proxy=httpx.Proxy(proxy) if proxy else None, + ) + + +def _env_proxy_map() -> dict[str, str | None]: + """The raw HTTPS_PROXY / NO_PROXY map httpx would have built. + + httpx only consults the proxy environment when `transport` is left unset + (`allow_env_proxies = trust_env and transport is None`), and we must pass a + transport for the IPv4 pin above. That combination silently drops proxy + support, so behind an egress proxy every upstream reachable ONLY through it + dies as a ReadTimeout — measured on celestrak.org, which took the satellite + layer to zero while the direct-routable feeds looked fine. + + A `None` value is a NO_PROXY host (loopback, so the localhost sidecars keep + bypassing the proxy). Returns empty when nothing is configured, which is the + unproxied default and leaves behaviour exactly as it was. + """ + try: + from httpx._utils import get_environment_proxies # noqa: PLC0415 + except ImportError: # pragma: no cover — private helper moved + return {} + return dict(get_environment_proxies()) + + +def _default_transport() -> httpx.AsyncBaseTransport: + """The pool when UPSTREAM_PROXIES is set, otherwise the plain transport.""" + from app import upstream_proxy # noqa: PLC0415 — avoids a config import cycle + from app.config import get_settings # noqa: PLC0415 + + settings = get_settings() + pool = upstream_proxy.build( + settings.upstream_proxies, + _transport, + cooldown_s=settings.upstream_proxy_cooldown_s, + max_tries=settings.upstream_proxy_max_tries, + fallback_direct=settings.upstream_proxy_fallback_direct, + ) + return pool or _transport() + + def get_client() -> httpx.AsyncClient: global _CLIENT if _CLIENT is None: + from app import upstream_proxy # noqa: PLC0415 + + default = _default_transport() + env = _env_proxy_map() + pooled = isinstance(default, upstream_proxy.RotatingProxyTransport) + mounts: dict[str, httpx.AsyncBaseTransport | None] = {} + for pattern, url in env.items(): + # With a pool active, an explicit UPSTREAM_PROXIES outranks the + # ambient environment proxy: keeping the env's scheme mount (e.g. + # "https://") would shadow the default transport and the pool would + # never see a single request. Its NO_PROXY exclusions still apply — + # those are "do not proxy this host", which the pool must honour too. + if pooled and url is not None: + continue + mounts[pattern] = _transport(url) + if pooled: + # Same-host sidecars (:8090 ADS-B, :8093 AIS) must not ride the pool: + # an external hop cannot reach them and would publish their traffic. + # The env map already pins loopback when NO_PROXY is set; add it + # explicitly for the case where only UPSTREAM_PROXIES is configured. + for pattern in upstream_proxy.LOOPBACK_PATTERNS: + mounts.setdefault(pattern, _transport()) _CLIENT = httpx.AsyncClient( timeout=httpx.Timeout(15.0, connect=5.0), headers={"User-Agent": "osint-console/0.1"}, - # local_address pins outbound sockets to IPv4. Several upstreams - # (CloudFront-backed weathercam.digitraffic.fi, cwwp2.dot.ca.gov) - # publish AAAA records; on hosts with broken IPv6 egress httpx - # exhausts the v6 attempts and reports "All connection attempts - # failed" while curl quietly falls back. One retry absorbs - # transient resets on long-lived pooled connections. - transport=httpx.AsyncHTTPTransport(local_address="0.0.0.0", retries=1), + transport=default, + mounts=mounts, ) return _CLIENT +def proxy_stats() -> list[dict[str, object]] | None: + """Per-proxy health when a pool is active, else None. Credentials redacted.""" + from app import upstream_proxy # noqa: PLC0415 + + if _CLIENT is None: + return None + transport = _CLIENT._transport # noqa: SLF001 — httpx exposes no public getter + if isinstance(transport, upstream_proxy.RotatingProxyTransport): + return transport.stats() + return None + + T = TypeVar("T") # Bounded LRU cap — large enough to cover all live route keys + per-bbox/per-id diff --git a/apps/api/app/upstream_proxy.py b/apps/api/app/upstream_proxy.py new file mode 100644 index 00000000..979e4d23 --- /dev/null +++ b/apps/api/app/upstream_proxy.py @@ -0,0 +1,206 @@ +"""Rotating outbound proxy pool for the shared upstream client. + +WHY: several upstreams gate on the CALLER'S IP, not on credentials — +airplanes.live and globe.theairtraffic.com answer a datacenter address with a +Cloudflare 403 (measured), adsb.lol rate-limits per source IP, and OpenSky's +anonymous credit budget is per IP. An operator who has their own egress +addresses can spread the load across them instead of burning one. + +SCOPE, deliberately: this pool is whatever the operator configures in +`UPSTREAM_PROXIES` and nothing else. There is no discovery, no scraping of +public "free proxy" lists, no bundled defaults. Those lists are not a supply of +spare capacity — the residential entries are overwhelmingly consumer machines +enrolled by malware whose owners never agreed to carry traffic, and the rest +are interception points that would read every query this console makes. Both +are somebody else's problem to be handed, not ours to hand them. Point this at +addresses you own or pay for. + +OFF unless configured: with `UPSTREAM_PROXIES` empty, `pool()` returns None and +`app.upstream` wires exactly the transport it always did. + +Failure handling is the point of the rotation. A proxy that errors is put on a +cooldown (`UPSTREAM_PROXY_COOLDOWN_S`) rather than dropped for good, since the +usual cause is a transient upstream refusal; the next request skips it. When +every entry is cooling down the pool either fails the request or falls back to +a direct connection, per `UPSTREAM_PROXY_FALLBACK_DIRECT` — default FALSE, +because an operator running through a proxy generally means "do not send this +from my own address", and silently reverting to direct would leak the very +thing the proxy was for. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from itertools import count + +import httpx + +log = logging.getLogger(__name__) + +# Loopback must never ride the pool: the ADS-B (:8090) and AIS (:8093) sidecars +# are same-host services, and routing them through an external hop would both +# break them and publish their traffic. app.upstream mounts these direct. +LOOPBACK_PATTERNS = ( + "all://localhost", + "all://127.0.0.1", + "all://[::1]", + "all://127.0.0.0/8", +) + + +def parse_pool(raw: str) -> list[str]: + """Split the configured list, keeping order and dropping blanks/dupes.""" + seen: set[str] = set() + out: list[str] = [] + for item in raw.split(","): + url = item.strip() + if not url or url in seen: + continue + seen.add(url) + out.append(url) + return out + + +class _Entry: + __slots__ = ("url", "transport", "cooldown_until", "failures", "successes") + + def __init__(self, url: str, transport: httpx.AsyncBaseTransport) -> None: + self.url = url + self.transport = transport + self.cooldown_until = 0.0 + self.failures = 0 + self.successes = 0 + + def available(self, now: float) -> bool: + return now >= self.cooldown_until + + +class RotatingProxyTransport(httpx.AsyncBaseTransport): + """Round-robins requests over the configured proxies, with failover. + + Implemented as a transport rather than per-call plumbing so every existing + `get_client().get(...)` call site is covered without being touched. + """ + + def __init__( + self, + entries: list[_Entry], + *, + direct: httpx.AsyncBaseTransport | None, + cooldown_s: float, + max_tries: int, + ) -> None: + if not entries: + raise ValueError("RotatingProxyTransport needs at least one proxy") + self._entries = entries + self._direct = direct + self._cooldown_s = cooldown_s + self._max_tries = max(1, min(max_tries, len(entries))) + self._counter = count() + + def _ordered(self, now: float) -> list[_Entry]: + """Available entries, starting at the rotation cursor.""" + n = len(self._entries) + start = next(self._counter) % n + rotated = [self._entries[(start + i) % n] for i in range(n)] + return [e for e in rotated if e.available(now)] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + now = time.monotonic() + candidates = self._ordered(now) + last_exc: Exception | None = None + + for entry in candidates[: self._max_tries]: + try: + response = await entry.transport.handle_async_request(request) + except Exception as exc: # noqa: BLE001 — any transport error rotates + last_exc = exc + entry.failures += 1 + entry.cooldown_until = time.monotonic() + self._cooldown_s + log.warning( + "upstream proxy %s failed (%s: %s), cooling down %.0fs", + entry.url, + type(exc).__name__, + exc, + self._cooldown_s, + ) + continue + entry.successes += 1 + return response + + if self._direct is not None: + log.warning( + "all %d upstream proxies unavailable — falling back to a direct " + "connection for %s", + len(self._entries), + request.url.host, + ) + return await self._direct.handle_async_request(request) + + # Fail closed. Raising the real error keeps the caller's existing + # httpx-exception handling working instead of inventing a new type. + if last_exc is not None: + raise last_exc + raise httpx.ConnectError( + "every configured upstream proxy is cooling down", request=request + ) + + def stats(self) -> list[dict[str, object]]: + """Per-proxy health, for /api/status surfaces. No credentials echoed.""" + now = time.monotonic() + return [ + { + "proxy": _redact(e.url), + "available": e.available(now), + "cooling_down_s": max(0.0, round(e.cooldown_until - now, 1)), + "successes": e.successes, + "failures": e.failures, + } + for e in self._entries + ] + + async def aclose(self) -> None: + for entry in self._entries: + await entry.transport.aclose() + if self._direct is not None: + await self._direct.aclose() + + +def _redact(url: str) -> str: + """Strip user:pass from a proxy URL so stats/logs never carry credentials.""" + try: + parsed = httpx.URL(url) + except Exception: # noqa: BLE001 — never let redaction raise + return "" + if parsed.username or parsed.password: + return str(parsed.copy_with(username="***", password="***")) + return str(parsed) + + +def build( + raw: str, + transport_factory: Callable[[str | None], httpx.AsyncBaseTransport], + *, + cooldown_s: float, + max_tries: int, + fallback_direct: bool, +) -> RotatingProxyTransport | None: + """Build the pool, or None when nothing is configured (the default).""" + urls = parse_pool(raw) + if not urls: + return None + entries = [_Entry(url, transport_factory(url)) for url in urls] + log.info( + "upstream proxy pool: %d configured (%s), fallback_direct=%s", + len(entries), + ", ".join(_redact(u) for u in urls), + fallback_direct, + ) + return RotatingProxyTransport( + entries, + direct=transport_factory(None) if fallback_direct else None, + cooldown_s=cooldown_s, + max_tries=max_tries, + ) diff --git a/apps/api/tests/test_upstream_proxy.py b/apps/api/tests/test_upstream_proxy.py new file mode 100644 index 00000000..5c41682f --- /dev/null +++ b/apps/api/tests/test_upstream_proxy.py @@ -0,0 +1,290 @@ +"""Rotating upstream proxy pool (app.upstream_proxy). + +The invariants worth guarding: OFF by default, rotation actually rotates, a +failing proxy is skipped rather than retried into the ground, the pool fails +CLOSED unless the operator opts into a direct fallback, loopback never rides +the pool, and credentials never reach stats or logs. +""" + +from __future__ import annotations + +import httpx +import pytest + +from app import upstream, upstream_proxy + + +def _factory(record: list[str | None]): + """transport_factory stand-in that records what it was asked to build.""" + + def make(proxy: str | None) -> httpx.AsyncBaseTransport: + record.append(proxy) + return httpx.MockTransport(lambda req: httpx.Response(200, text=str(proxy))) + + return make + + +class _Boom(httpx.AsyncBaseTransport): + """Always fails, so a pool member can be driven into cooldown.""" + + def __init__(self) -> None: + self.calls = 0 + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.calls += 1 + raise httpx.ConnectError("nope", request=request) + + +class _Ok(httpx.AsyncBaseTransport): + def __init__(self, tag: str) -> None: + self.tag = tag + self.calls = 0 + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.calls += 1 + return httpx.Response(200, text=self.tag) + + +def _pool(entries, *, direct=None, cooldown_s=300.0, max_tries=3): + return upstream_proxy.RotatingProxyTransport( + [upstream_proxy._Entry(tag, tr) for tag, tr in entries], + direct=direct, + cooldown_s=cooldown_s, + max_tries=max_tries, + ) + + +def test_parse_pool_drops_blanks_and_dupes_keeping_order() -> None: + raw = " http://a:1 , ,http://b:2, http://a:1 ," + assert upstream_proxy.parse_pool(raw) == ["http://a:1", "http://b:2"] + + +def test_build_returns_none_when_unconfigured() -> None: + """Empty config must leave the client exactly as it was — the default.""" + record: list[str | None] = [] + assert ( + upstream_proxy.build( + "", _factory(record), cooldown_s=1.0, max_tries=3, fallback_direct=False + ) + is None + ) + assert record == [] + + +def test_build_makes_one_transport_per_proxy() -> None: + record: list[str | None] = [] + pool = upstream_proxy.build( + "http://a:1,http://b:2", + _factory(record), + cooldown_s=1.0, + max_tries=3, + fallback_direct=False, + ) + assert pool is not None + assert record == ["http://a:1", "http://b:2"] + + +def test_build_only_makes_a_direct_transport_when_fallback_enabled() -> None: + record: list[str | None] = [] + upstream_proxy.build( + "http://a:1", _factory(record), cooldown_s=1.0, max_tries=3, fallback_direct=True + ) + assert None in record, "fallback_direct=True must build an unproxied transport" + + record.clear() + upstream_proxy.build( + "http://a:1", + _factory(record), + cooldown_s=1.0, + max_tries=3, + fallback_direct=False, + ) + assert None not in record, "default must not build a direct escape hatch" + + +@pytest.mark.asyncio +async def test_requests_rotate_across_the_pool() -> None: + a, b = _Ok("a"), _Ok("b") + pool = _pool([("http://a:1", a), ("http://b:2", b)]) + req = httpx.Request("GET", "https://example.com") + + for _ in range(4): + await pool.handle_async_request(req) + + assert a.calls == 2 and b.calls == 2, "round-robin should split evenly" + + +@pytest.mark.asyncio +async def test_failing_proxy_fails_over_then_is_skipped_while_cooling() -> None: + bad, good = _Boom(), _Ok("good") + pool = _pool([("http://bad:1", bad), ("http://good:2", good)], cooldown_s=300.0) + req = httpx.Request("GET", "https://example.com") + + r = await pool.handle_async_request(req) + assert r.text == "good", "a dead proxy must fail over, not surface the error" + + before = bad.calls + for _ in range(6): + await pool.handle_async_request(req) + assert bad.calls == before, "a cooling-down proxy must not be retried" + assert good.calls >= 6 + + +@pytest.mark.asyncio +async def test_cooldown_expiry_puts_a_proxy_back_in_rotation() -> None: + bad, good = _Boom(), _Ok("good") + pool = _pool([("http://bad:1", bad), ("http://good:2", good)], cooldown_s=0.0) + req = httpx.Request("GET", "https://example.com") + + for _ in range(3): + await pool.handle_async_request(req) + assert bad.calls > 1, "a zero cooldown should let the proxy be tried again" + + +@pytest.mark.asyncio +async def test_pool_fails_closed_when_everything_is_down() -> None: + """The IP-leak guard: no direct fallback unless explicitly configured.""" + pool = _pool([("http://a:1", _Boom()), ("http://b:2", _Boom())], direct=None) + req = httpx.Request("GET", "https://example.com") + + with pytest.raises(httpx.ConnectError): + await pool.handle_async_request(req) + + +@pytest.mark.asyncio +async def test_direct_fallback_used_only_when_supplied() -> None: + direct = _Ok("direct") + pool = _pool([("http://a:1", _Boom())], direct=direct) + req = httpx.Request("GET", "https://example.com") + + r = await pool.handle_async_request(req) + assert r.text == "direct" + assert direct.calls == 1 + + +@pytest.mark.asyncio +async def test_max_tries_bounds_the_failover_walk() -> None: + booms = [_Boom() for _ in range(5)] + pool = _pool( + [(f"http://p{i}:1", b) for i, b in enumerate(booms)], direct=None, max_tries=2 + ) + req = httpx.Request("GET", "https://example.com") + + with pytest.raises(httpx.ConnectError): + await pool.handle_async_request(req) + assert sum(b.calls for b in booms) == 2, "must stop after max_tries proxies" + + +@pytest.mark.asyncio +async def test_stats_redact_credentials() -> None: + pool = _pool([("http://user:hunter2@host:8080", _Ok("a"))]) + stats = pool.stats() + + assert len(stats) == 1 + rendered = str(stats[0]) + assert "hunter2" not in rendered, "proxy password must never reach stats" + assert "user" not in rendered + assert "host:8080" in rendered + + +def test_redact_never_raises_on_junk() -> None: + assert isinstance(upstream_proxy._redact("::::not a url::::"), str) + + +def test_loopback_patterns_cover_the_sidecar_hosts() -> None: + """The :8090 / :8093 sidecars are same-host and must bypass any pool.""" + joined = " ".join(upstream_proxy.LOOPBACK_PATTERNS) + assert "127.0.0.1" in joined + assert "localhost" in joined + + +def test_client_pins_loopback_direct_when_a_pool_is_active(monkeypatch) -> None: + from app.config import get_settings + + settings = get_settings() + + monkeypatch.setattr(settings, "upstream_proxies", "http://proxy.invalid:8080") + monkeypatch.setattr(upstream, "_CLIENT", None) + try: + client = upstream.get_client() + assert isinstance(client._transport, upstream_proxy.RotatingProxyTransport) + # _mounts is keyed by httpx's private URLPattern; compare on its text. + mounted = {str(k.pattern): v for k, v in client._mounts.items()} + for pattern in upstream_proxy.LOOPBACK_PATTERNS: + assert pattern in mounted, f"{pattern} must be mounted" + assert not isinstance( + mounted[pattern], upstream_proxy.RotatingProxyTransport + ), f"{pattern} must bypass the pool" + finally: + upstream._CLIENT = None + + +def test_pool_outranks_the_environment_proxy(monkeypatch) -> None: + """Regression: an env scheme mount used to shadow the pool entirely. + + httpx consults `mounts` before the default transport, so with HTTPS_PROXY + set the "https://" entry answered every request and the configured pool + never saw one — measured live: four 200s with 0 successes recorded. The + explicit UPSTREAM_PROXIES must win; NO_PROXY exclusions must still apply. + """ + from app.config import get_settings + + settings = get_settings() + + # httpx reads the lowercase names first, so set both or the ambient + # lowercase values (which this sandbox sets) silently win. + for name in ("HTTPS_PROXY", "https_proxy"): + monkeypatch.setenv(name, "http://env-proxy.invalid:3128") + for name in ("NO_PROXY", "no_proxy"): + monkeypatch.setenv(name, "skip-me.invalid") + monkeypatch.setattr(settings, "upstream_proxies", "http://pool.invalid:8080") + monkeypatch.setattr(upstream, "_CLIENT", None) + try: + client = upstream.get_client() + mounted = {str(k.pattern): v for k, v in client._mounts.items()} + + assert "https://" not in mounted, ( + "the env scheme mount must not shadow the pool" + ) + assert isinstance(client._transport, upstream_proxy.RotatingProxyTransport) + # NO_PROXY is an exclusion, not a route — the pool must honour it. + assert any("skip-me.invalid" in p for p in mounted), ( + "NO_PROXY exclusions must survive" + ) + finally: + upstream._CLIENT = None + + +def test_env_proxy_still_mounted_when_no_pool(monkeypatch) -> None: + """Without a pool the env proxy must still route — the celestrak fix.""" + from app.config import get_settings + + settings = get_settings() + + for name in ("HTTPS_PROXY", "https_proxy"): + monkeypatch.setenv(name, "http://env-proxy.invalid:3128") + monkeypatch.setattr(settings, "upstream_proxies", "") + monkeypatch.setattr(upstream, "_CLIENT", None) + try: + client = upstream.get_client() + mounted = {str(k.pattern): v for k, v in client._mounts.items()} + assert "https://" in mounted, "env proxy must route when no pool is set" + finally: + upstream._CLIENT = None + + +def test_client_is_unchanged_when_no_pool_configured(monkeypatch) -> None: + from app.config import get_settings + + settings = get_settings() + + monkeypatch.setattr(settings, "upstream_proxies", "") + monkeypatch.setattr(upstream, "_CLIENT", None) + try: + client = upstream.get_client() + assert not isinstance( + client._transport, upstream_proxy.RotatingProxyTransport + ), "an empty pool must not wrap the transport" + assert upstream.proxy_stats() is None + finally: + upstream._CLIENT = None diff --git a/apps/web/src/settings/localAi/AiSetupWizard.tsx b/apps/web/src/settings/localAi/AiSetupWizard.tsx index fa756811..f84059d3 100644 --- a/apps/web/src/settings/localAi/AiSetupWizard.tsx +++ b/apps/web/src/settings/localAi/AiSetupWizard.tsx @@ -131,9 +131,9 @@ export function AiSetupWizard({ onClose }: { onClose: () => void }): JSX.Element -
{p.repo_id}
+
{p.repo_id ?? '—'}
- {p.quant} · ~{p.est_size_gb.toFixed(0)} GB + {p.quant ?? '—'} · {p.est_size_gb === null ? '—' : `~${p.est_size_gb.toFixed(0)} GB`}

{recommended ? hardware.recommendation.reason : p.reason} @@ -205,17 +205,20 @@ function ConfirmStep({ diskFreeMb: number; error: string | null; }): JSX.Element { - const estBytes = preset.est_size_gb * 1024 * 1024 * 1024; + // A preset that does not fit reports no size, so there is nothing to compare + // against free disk — skip the warning rather than guess at a figure. const freeBytes = diskFreeMb * 1024 * 1024; - const tight = estBytes > freeBytes * 0.8; + const tight = + preset.est_size_gb !== null && preset.est_size_gb * 1024 * 1024 * 1024 > freeBytes * 0.8; return (

Confirm download:

-
{preset.repo_id}
-
{preset.quant}
+
{preset.repo_id ?? '—'}
+
{preset.quant ?? '—'}
- ~{preset.est_size_gb.toFixed(1)} GB to download · {humanBytes(diskFreeMb * 1024 * 1024)} free + {preset.est_size_gb === null ? '—' : `~${preset.est_size_gb.toFixed(1)} GB`} to download ·{' '} + {humanBytes(diskFreeMb * 1024 * 1024)} free
{tight && ( diff --git a/apps/web/src/settings/localAi/PresetsRow.tsx b/apps/web/src/settings/localAi/PresetsRow.tsx index 221033c4..08cb7bdc 100644 --- a/apps/web/src/settings/localAi/PresetsRow.tsx +++ b/apps/web/src/settings/localAi/PresetsRow.tsx @@ -55,6 +55,10 @@ function PresetCard({ onDownload: (repoId: string, quant: string) => void; }): JSX.Element { const refused = !preset.fits && Boolean(preset.refused_reason); + // Consts, not property reads: narrowing a property does not survive into the + // onClick closure below, so the download call would not typecheck. + const repoId = preset.repo_id; + const quant = preset.quant; const already = installed.some((m) => m.repo_id === preset.repo_id && m.quant === preset.quant); const [settingActive, setSettingActive] = useState(false); @@ -83,19 +87,22 @@ function PresetCard({ {PRESET_LABEL[id]} {recommended && recommended}
-
{preset.tier} tier
-
- {preset.repo_id} +
{preset.tier ?? '—'} tier
+
+ {preset.repo_id ?? '—'}
- {preset.quant} · ~{preset.est_size_gb.toFixed(0)} GB + {preset.quant ?? '—'} ·{' '} + {preset.est_size_gb === null ? '—' : `~${preset.est_size_gb.toFixed(0)} GB`}

{recommended ? recommendationReason : preset.reason}

{refused &&

{preset.refused_reason}

}
- {refused ? ( + {/* No catalog entry fit this hardware, so there is no repo/quant to + pull — the card offers nothing to download, same as a refusal. */} + {refused || repoId === null || quant === null ? ( unavailable @@ -104,7 +111,7 @@ function PresetCard({ {settingActive ? '…' : 'Set as main'} ) : ( - onDownload(preset.repo_id, preset.quant)} className="w-full"> + onDownload(repoId, quant)} className="w-full"> ⭳ Download & use )} diff --git a/apps/web/src/settings/localAi/types.ts b/apps/web/src/settings/localAi/types.ts index 56348503..07511139 100644 --- a/apps/web/src/settings/localAi/types.ts +++ b/apps/web/src/settings/localAi/types.ts @@ -70,11 +70,16 @@ export interface HardwareGpu { vram_mb: number; } +// A preset the hardware cannot run carries NO catalog entry: the backend's +// `_preset_dict(entry=None, ...)` nulls tier/repo_id/quant/est_size_gb and sets +// fits=false (app/localllm/hardware.py). Typing these non-nullable is what let +// `est_size_gb.toFixed()` take the whole shell down on any box where a tier +// does not fit (no GPU → the speed preset is always null). export interface HardwarePreset { - tier: string; - repo_id: string; - quant: string; - est_size_gb: number; + tier: string | null; + repo_id: string | null; + quant: string | null; + est_size_gb: number | null; fits: boolean; reason: string; refused_reason?: string; diff --git a/docs/decisions.md b/docs/decisions.md index a202e3f5..f8a09fc8 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -741,6 +741,7 @@ Findings from the same-day state-of-project audit ## Backend test baseline history +- 2108 + 2 skipped — 2026-07-30, overnight-provenance-answers-2026-07-29 - 1985 + 2 skipped — 2026-07-27, perf-annotate-sidecars-2026-07-27, performance wave - 1972 + 2 skipped — 2026-07-24, worldmonitor-gaps-2026-07, persona waves 2+3