Stop a preset that does not fit from taking the whole shell down

.env.example

@@ -78,6 +78,28 @@ RECON_JOB_TTL_S=86400
 # Disk tile cache root — safe to delete anytime, refills on demand.
 TILE_CACHE_DIR=./data/tilecache
 
+# ── Outbound proxy pool (optional; empty = off) ──────────────────────────────
+# Comma-separated proxies the shared upstream client rotates over. Useful
+# because several upstreams gate on the CALLER'S IP rather than a key:
+# airplanes.live 403s a datacenter address, adsb.lol rate-limits per source IP,
+# and OpenSky's anonymous credit budget is per IP.
+#   UPSTREAM_PROXIES=http://user:pass@host:8080,socks5://host:1080
+# Point this ONLY at egress you own or pay for. There is deliberately no
+# discovery and no bundled list: public "free proxy" pools are largely consumer
+# machines enrolled by malware whose owners never agreed to carry traffic, plus
+# interception nodes that would read every query this console makes.
+# An explicit pool here outranks HTTPS_PROXY; NO_PROXY exclusions still apply,
+# and the localhost sidecars (:8090, :8093) always bypass the pool.
+UPSTREAM_PROXIES=
+# A proxy that errors is skipped for this long before being retried.
+UPSTREAM_PROXY_COOLDOWN_S=300
+# How many pool members one request may try before giving up.
+UPSTREAM_PROXY_MAX_TRIES=3
+# When every proxy is cooling down: 0 (default) fails the request, 1 sends it
+# directly. Default 0 on purpose — configuring a proxy usually means "not from
+# my own address", and a silent direct fallback would leak exactly that.
+UPSTREAM_PROXY_FALLBACK_DIRECT=
+
 # ── MCP server + local AI (Ollama) ───────────────────────────────────────────
 API_BASE=http://localhost:8000
 OLLAMA_HOST=http://localhost:11434

CLAUDE.md

@@ -155,9 +155,9 @@ Ontology (2026-07-07, docs/decisions.md#ontology-local-first-store-2026-07-07):
 - Backend tests from the **repo ROOT** (from `apps/api` the `.env` auth
   resolves → wall of 401s):
   `OSINT_DISABLE_BACKGROUND=1 apps/api/.venv/bin/pytest apps/api -q`
-  Baseline: **2108 passed + 2 skipped in ~110 s** (skip = opt-in live probes;
-  measured 2026-07-30, branch overnight-provenance-answers-2026-07-29 — see
-  `docs/exec-report-2026-07-29.md`). Runs SERIAL by default: `-n auto --dist
+  Baseline: **2141 passed + 2 skipped in ~333 s** (skip = opt-in live probes;
+  measured 2026-07-31, branch repo-setup-ui-access, upstream proxy pool wave).
+  Runs SERIAL by default: `-n auto --dist
   loadfile` groups different files per worker on different core counts, so a
   suite with module-state leaks answers differently per machine and CI (4 cores)
   failed a branch that was green locally (16). Opt in per-machine, never commit

apps/api/app/config.py

@@ -127,6 +127,28 @@ class Settings(BaseSettings):
     # union for deploys without a sidecar; the local .env sets ADSB_SIDECAR_ONLY=1.
     adsb_sidecar_only: bool = False
 
+    # ── outbound proxy pool (app.upstream_proxy) ──
+    # Comma-separated proxy URLs the shared upstream client rotates over, e.g.
+    # "http://user:pass@host:8080,socks5://host:1080". EMPTY = off, which is the
+    # default and leaves the client wired exactly as it was.
+    # Point this ONLY at egress you own or pay for. There is no discovery and no
+    # bundled list: public "free proxy" pools are mostly malware-enrolled home
+    # machines and interception points, so they would route this console's
+    # queries through devices whose owners never agreed to carry them.
+    # Useful because several upstreams gate on the caller's IP rather than a
+    # key — airplanes.live 403s a datacenter address, adsb.lol rate-limits per
+    # source IP, OpenSky's anonymous credit budget is per IP.
+    upstream_proxies: str = ""  # UPSTREAM_PROXIES
+    # A proxy that errors is skipped for this long before being retried.
+    upstream_proxy_cooldown_s: float = 300.0
+    # How many pool members one request may try before giving up.
+    upstream_proxy_max_tries: int = 3
+    # When every proxy is cooling down: False (default) fails the request, True
+    # sends it directly. Default False on purpose — configuring a proxy usually
+    # means "not from my own address", and a silent direct fallback would leak
+    # exactly what the proxy was there to avoid.
+    upstream_proxy_fallback_direct: bool = False
+
     # ── infra ──
     database_url: str = "postgresql+asyncpg://osint:osint@localhost:5432/osint"
     redis_url: str = "redis://localhost:6379/0"

apps/api/app/upstream.py

@@ -18,23 +18,106 @@
 _CLIENT: httpx.AsyncClient | None = None
 
 
+def _transport(proxy: str | None = None) -> httpx.AsyncHTTPTransport:
+    """One transport shape, optionally routed through `proxy`.
+
+    local_address pins outbound sockets to IPv4. Several upstreams
+    (CloudFront-backed weathercam.digitraffic.fi, cwwp2.dot.ca.gov) publish
+    AAAA records; on hosts with broken IPv6 egress httpx exhausts the v6
+    attempts and reports "All connection attempts failed" while curl quietly
+    falls back. One retry absorbs transient resets on long-lived pooled
+    connections.
+    """
+    return httpx.AsyncHTTPTransport(
+        local_address="0.0.0.0",
+        retries=1,
+        proxy=httpx.Proxy(proxy) if proxy else None,
+    )
+
+
+def _env_proxy_map() -> dict[str, str | None]:
+    """The raw HTTPS_PROXY / NO_PROXY map httpx would have built.
+
+    httpx only consults the proxy environment when `transport` is left unset
+    (`allow_env_proxies = trust_env and transport is None`), and we must pass a
+    transport for the IPv4 pin above. That combination silently drops proxy
+    support, so behind an egress proxy every upstream reachable ONLY through it
+    dies as a ReadTimeout — measured on celestrak.org, which took the satellite
+    layer to zero while the direct-routable feeds looked fine.
+
+    A `None` value is a NO_PROXY host (loopback, so the localhost sidecars keep
+    bypassing the proxy). Returns empty when nothing is configured, which is the
+    unproxied default and leaves behaviour exactly as it was.
+    """
+    try:
+        from httpx._utils import get_environment_proxies  # noqa: PLC0415
+    except ImportError:  # pragma: no cover — private helper moved
+        return {}
+    return dict(get_environment_proxies())
+
+
+def _default_transport() -> httpx.AsyncBaseTransport:
+    """The pool when UPSTREAM_PROXIES is set, otherwise the plain transport."""
+    from app import upstream_proxy  # noqa: PLC0415 — avoids a config import cycle
+    from app.config import get_settings  # noqa: PLC0415
+
+    settings = get_settings()
+    pool = upstream_proxy.build(
+        settings.upstream_proxies,
+        _transport,
+        cooldown_s=settings.upstream_proxy_cooldown_s,
+        max_tries=settings.upstream_proxy_max_tries,
+        fallback_direct=settings.upstream_proxy_fallback_direct,
+    )
+    return pool or _transport()
+
+
 def get_client() -> httpx.AsyncClient:
     global _CLIENT
     if _CLIENT is None:
+        from app import upstream_proxy  # noqa: PLC0415
+
+        default = _default_transport()
+        env = _env_proxy_map()
+        pooled = isinstance(default, upstream_proxy.RotatingProxyTransport)
+        mounts: dict[str, httpx.AsyncBaseTransport | None] = {}
+        for pattern, url in env.items():
+            # With a pool active, an explicit UPSTREAM_PROXIES outranks the
+            # ambient environment proxy: keeping the env's scheme mount (e.g.
+            # "https://") would shadow the default transport and the pool would
+            # never see a single request. Its NO_PROXY exclusions still apply —
+            # those are "do not proxy this host", which the pool must honour too.
+            if pooled and url is not None:
+                continue
+            mounts[pattern] = _transport(url)
+        if pooled:
+            # Same-host sidecars (:8090 ADS-B, :8093 AIS) must not ride the pool:
+            # an external hop cannot reach them and would publish their traffic.
+            # The env map already pins loopback when NO_PROXY is set; add it
+            # explicitly for the case where only UPSTREAM_PROXIES is configured.
+            for pattern in upstream_proxy.LOOPBACK_PATTERNS:
+                mounts.setdefault(pattern, _transport())
         _CLIENT = httpx.AsyncClient(
             timeout=httpx.Timeout(15.0, connect=5.0),
             headers={"User-Agent": "osint-console/0.1"},
-            # local_address pins outbound sockets to IPv4. Several upstreams
-            # (CloudFront-backed weathercam.digitraffic.fi, cwwp2.dot.ca.gov)
-            # publish AAAA records; on hosts with broken IPv6 egress httpx
-            # exhausts the v6 attempts and reports "All connection attempts
-            # failed" while curl quietly falls back. One retry absorbs
-            # transient resets on long-lived pooled connections.
-            transport=httpx.AsyncHTTPTransport(local_address="0.0.0.0", retries=1),
+            transport=default,
+            mounts=mounts,
         )
     return _CLIENT
 
 
+def proxy_stats() -> list[dict[str, object]] | None:
+    """Per-proxy health when a pool is active, else None. Credentials redacted."""
+    from app import upstream_proxy  # noqa: PLC0415
+
+    if _CLIENT is None:
+        return None
+    transport = _CLIENT._transport  # noqa: SLF001 — httpx exposes no public getter
+    if isinstance(transport, upstream_proxy.RotatingProxyTransport):
+        return transport.stats()
+    return None
+
+
 T = TypeVar("T")
 
 # Bounded LRU cap — large enough to cover all live route keys + per-bbox/per-id

apps/api/app/upstream_proxy.py

@@ -0,0 +1,206 @@
+"""Rotating outbound proxy pool for the shared upstream client.
+
+WHY: several upstreams gate on the CALLER'S IP, not on credentials —
+airplanes.live and globe.theairtraffic.com answer a datacenter address with a
+Cloudflare 403 (measured), adsb.lol rate-limits per source IP, and OpenSky's
+anonymous credit budget is per IP. An operator who has their own egress
+addresses can spread the load across them instead of burning one.
+
+SCOPE, deliberately: this pool is whatever the operator configures in
+`UPSTREAM_PROXIES` and nothing else. There is no discovery, no scraping of
+public "free proxy" lists, no bundled defaults. Those lists are not a supply of
+spare capacity — the residential entries are overwhelmingly consumer machines
+enrolled by malware whose owners never agreed to carry traffic, and the rest
+are interception points that would read every query this console makes. Both
+are somebody else's problem to be handed, not ours to hand them. Point this at
+addresses you own or pay for.
+
+OFF unless configured: with `UPSTREAM_PROXIES` empty, `pool()` returns None and
+`app.upstream` wires exactly the transport it always did.
+
+Failure handling is the point of the rotation. A proxy that errors is put on a
+cooldown (`UPSTREAM_PROXY_COOLDOWN_S`) rather than dropped for good, since the
+usual cause is a transient upstream refusal; the next request skips it. When
+every entry is cooling down the pool either fails the request or falls back to
+a direct connection, per `UPSTREAM_PROXY_FALLBACK_DIRECT` — default FALSE,
+because an operator running through a proxy generally means "do not send this
+from my own address", and silently reverting to direct would leak the very
+thing the proxy was for.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable
+from itertools import count
+
+import httpx
+
+log = logging.getLogger(__name__)
+
+# Loopback must never ride the pool: the ADS-B (:8090) and AIS (:8093) sidecars
+# are same-host services, and routing them through an external hop would both
+# break them and publish their traffic. app.upstream mounts these direct.
+LOOPBACK_PATTERNS = (
+    "all://localhost",
+    "all://127.0.0.1",
+    "all://[::1]",
+    "all://127.0.0.0/8",
+)
+
+
+def parse_pool(raw: str) -> list[str]:
+    """Split the configured list, keeping order and dropping blanks/dupes."""
+    seen: set[str] = set()
+    out: list[str] = []
+    for item in raw.split(","):
+        url = item.strip()
+        if not url or url in seen:
+            continue
+        seen.add(url)
+        out.append(url)
+    return out
+
+
+class _Entry:
+    __slots__ = ("url", "transport", "cooldown_until", "failures", "successes")
+
+    def __init__(self, url: str, transport: httpx.AsyncBaseTransport) -> None:
+        self.url = url
+        self.transport = transport
+        self.cooldown_until = 0.0
+        self.failures = 0
+        self.successes = 0
+
+    def available(self, now: float) -> bool:
+        return now >= self.cooldown_until
+
+
+class RotatingProxyTransport(httpx.AsyncBaseTransport):
+    """Round-robins requests over the configured proxies, with failover.
+
+    Implemented as a transport rather than per-call plumbing so every existing
+    `get_client().get(...)` call site is covered without being touched.
+    """
+
+    def __init__(
+        self,
+        entries: list[_Entry],
+        *,
+        direct: httpx.AsyncBaseTransport | None,
+        cooldown_s: float,
+        max_tries: int,
+    ) -> None:
+        if not entries:
+            raise ValueError("RotatingProxyTransport needs at least one proxy")
+        self._entries = entries
+        self._direct = direct
+        self._cooldown_s = cooldown_s
+        self._max_tries = max(1, min(max_tries, len(entries)))
+        self._counter = count()
+
+    def _ordered(self, now: float) -> list[_Entry]:
+        """Available entries, starting at the rotation cursor."""
+        n = len(self._entries)
+        start = next(self._counter) % n
+        rotated = [self._entries[(start + i) % n] for i in range(n)]
+        return [e for e in rotated if e.available(now)]
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        now = time.monotonic()
+        candidates = self._ordered(now)
+        last_exc: Exception | None = None
+
+        for entry in candidates[: self._max_tries]:
+            try:
+                response = await entry.transport.handle_async_request(request)
+            except Exception as exc:  # noqa: BLE001 — any transport error rotates
+                last_exc = exc
+                entry.failures += 1
+                entry.cooldown_until = time.monotonic() + self._cooldown_s
+                log.warning(
+                    "upstream proxy %s failed (%s: %s), cooling down %.0fs",
+                    entry.url,
+                    type(exc).__name__,
+                    exc,
+                    self._cooldown_s,
+                )
+                continue
+            entry.successes += 1
+            return response
+
+        if self._direct is not None:
+            log.warning(
+                "all %d upstream proxies unavailable — falling back to a direct "
+                "connection for %s",
+                len(self._entries),
+                request.url.host,
+            )
+            return await self._direct.handle_async_request(request)
+
+        # Fail closed. Raising the real error keeps the caller's existing
+        # httpx-exception handling working instead of inventing a new type.
+        if last_exc is not None:
+            raise last_exc
+        raise httpx.ConnectError(
+            "every configured upstream proxy is cooling down", request=request
+        )
+
+    def stats(self) -> list[dict[str, object]]:
+        """Per-proxy health, for /api/status surfaces. No credentials echoed."""
+        now = time.monotonic()
+        return [
+            {
+                "proxy": _redact(e.url),
+                "available": e.available(now),
+                "cooling_down_s": max(0.0, round(e.cooldown_until - now, 1)),
+                "successes": e.successes,
+                "failures": e.failures,
+            }
+            for e in self._entries
+        ]
+
+    async def aclose(self) -> None:
+        for entry in self._entries:
+            await entry.transport.aclose()
+        if self._direct is not None:
+            await self._direct.aclose()
+
+
+def _redact(url: str) -> str:
+    """Strip user:pass from a proxy URL so stats/logs never carry credentials."""
+    try:
+        parsed = httpx.URL(url)
+    except Exception:  # noqa: BLE001 — never let redaction raise
+        return "<unparseable>"
+    if parsed.username or parsed.password:
+        return str(parsed.copy_with(username="***", password="***"))
+    return str(parsed)
+
+
+def build(
+    raw: str,
+    transport_factory: Callable[[str | None], httpx.AsyncBaseTransport],
+    *,
+    cooldown_s: float,
+    max_tries: int,
+    fallback_direct: bool,
+) -> RotatingProxyTransport | None:
+    """Build the pool, or None when nothing is configured (the default)."""
+    urls = parse_pool(raw)
+    if not urls:
+        return None
+    entries = [_Entry(url, transport_factory(url)) for url in urls]
+    log.info(
+        "upstream proxy pool: %d configured (%s), fallback_direct=%s",
+        len(entries),
+        ", ".join(_redact(u) for u in urls),
+        fallback_direct,
+    )
+    return RotatingProxyTransport(
+        entries,
+        direct=transport_factory(None) if fallback_direct else None,
+        cooldown_s=cooldown_s,
+        max_tries=max_tries,
+    )