diff --git a/CLAUDE.md b/CLAUDE.md index 848e8e41..31a6e9fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,8 +52,8 @@ undo and why it was made. - Backend tests from the **repo ROOT** (from `apps/api` the `.env` auth resolves → wall of 401s): `OSINT_DISABLE_BACKGROUND=1 apps/api/.venv/bin/pytest apps/api -q` - Baseline: **2255 passed + 2 skipped in ~134 s** (skip = opt-in live probes; - measured 2026-08-08, branch gotham-console-mockup, feed-cadence fix). + Baseline: **2393 passed + 2 skipped in ~152 s** (skip = opt-in live probes; + measured 2026-08-08, branch gotham-parity-2026-08, connection wire coverage). Runs SERIAL by default: `-n auto --dist loadfile` groups different files per worker on different core counts, so a suite with module-state leaks answers differently per machine and CI (4 cores) diff --git a/apps/api/CLAUDE.md b/apps/api/CLAUDE.md index 18a18d00..cd213517 100644 --- a/apps/api/CLAUDE.md +++ b/apps/api/CLAUDE.md @@ -95,6 +95,38 @@ Browser-tier pacing and the headful lever are in `tools/CLAUDE.md`. WS handlers call `require_ws_key` BEFORE `accept`. +`POST /api/ingest/{dataset_id}` is the ONE route with no session dependency — an +external sender has no session, so a per-dataset token is the whole gate. Only +the token's sha256 is stored, comparison is `compare_digest`, the token is never +logged or echoed after the response that mints it, the body is capped BEFORE it +is parsed (Content-Length AND a running total, since chunked declares neither), +and an unknown dataset and an unarmed one answer with the identical 404 so the +route cannot enumerate dataset ids. → `tests/test_ingest_webhook.py` + +## Connections (operator-configured sources) + +`foundry/connections.py` runs MQTT / Kafka / SQL sources the operator points at +their own infrastructure. Two rules: + +- A `sql` connection stores the **NAME of an environment variable** holding the + DSN, never the DSN. The row is returned by the list route and sits in + `foundry.db`; a password in it is a leak with several copies. Driver + exceptions are scrubbed of the DSN before they reach `last_error`. +- `aiokafka` and `sqlalchemy` are OPTIONAL extras, import-guarded like + `titiler-core`. An absent client makes its kind report unavailable; it never + stops the app booting. The guard simulates absence with a `__import__` shim, + because once the extra is installed a test that merely imports proves nothing. + → `tests/test_connections.py` +- Both wire paths are proven against something real, not mocked: MQTT against a + forty-line asyncio broker in the test (`tests/test_mqtt_client.py`), SQL + against SQLite through SQLAlchemy (`tests/test_connections_sql.py`, which is + why `sqlalchemy` is also a DEV dependency). **Kafka has no equivalent** — it + needs a broker this box cannot run — so its runner is configured and + supervised but unproven on the wire. + +Supervised, not started once (same rule as the sidecars): the reconcile loop +restarts a connection that dies later and applies an edit made in the UI. + ## Ontology (2026-07-07, docs/decisions.md#ontology-local-first-store-2026-07-07) - The ONLY backend = local SQLite (`intel/ontology_local.py`, via diff --git a/apps/api/app/ais_keyless.py b/apps/api/app/ais_keyless.py index e54171e8..2b3de137 100644 --- a/apps/api/app/ais_keyless.py +++ b/apps/api/app/ais_keyless.py @@ -33,6 +33,17 @@ from app import ais_firehose from app.config import get_settings from app.correlate.types import Observation + +# The MQTT 3.1.1 wire codec now lives in ``app/mqtt_client.py`` so a +# user-configured broker connection can reuse it instead of growing a second +# implementation. Re-exported under the old private names: the guard test in +# tests/test_ais_keyless.py reaches them through this module, and keeping it +# pointed here is what proves the extraction changed no bytes. +from app.mqtt_client import connect_packet as _connect_packet +from app.mqtt_client import decode_publish as _decode_publish +from app.mqtt_client import enc_remaining_length as _enc_remaining_length # noqa: F401 +from app.mqtt_client import parse_packets as _parse_packets +from app.mqtt_client import subscribe_packet as _subscribe_packet from app.routes import ais as ais_routes from app.upstream import get_client @@ -157,76 +168,6 @@ async def _run_kystdatahuset() -> None: # ── Digitraffic (Finland/Baltic) — minimal MQTT 3.1.1 over WSS ───────────────── -def _enc_remaining_length(n: int) -> bytes: - out = bytearray() - while True: - b = n % 128 - n //= 128 - if n > 0: - b |= 0x80 - out.append(b) - if n == 0: - break - return bytes(out) - - -def _connect_packet(client_id: str = "osint-geoint") -> bytes: - # variable header: proto name "MQTT", level 4, clean-session flag, keepalive 60 - vh = b"\x00\x04MQTT\x04\x02\x00\x3c" - payload = len(client_id).to_bytes(2, "big") + client_id.encode() - body = vh + payload - return b"\x10" + _enc_remaining_length(len(body)) + body - - -def _subscribe_packet(topic: str, packet_id: int = 1) -> bytes: - body = packet_id.to_bytes(2, "big") + len(topic).to_bytes(2, "big") + topic.encode() + b"\x00" - return b"\x82" + _enc_remaining_length(len(body)) + body - - -def _parse_packets(buf: bytes) -> tuple[list[tuple[int, int, bytes]], bytes]: - """Parse complete MQTT packets from ``buf``. - - Returns ``([(packet_type, byte0, body), …], remainder)``. A WS frame may - carry partial / multiple MQTT packets, so the caller accumulates the - remainder across reads. - """ - out: list[tuple[int, int, bytes]] = [] - i, n = 0, len(buf) - while i < n: - b0 = buf[i] - ptype = b0 >> 4 - mult, rl, j = 1, 0, i + 1 - while True: - if j >= n: - return out, buf[i:] # length incomplete - d = buf[j] - rl += (d & 0x7F) * mult - mult *= 128 - j += 1 - if not (d & 0x80): - break - if mult > 128**4: - return out, b"" # malformed; drop - if j + rl > n: - return out, buf[i:] # body incomplete - out.append((ptype, b0, buf[j : j + rl])) - i = j + rl - return out, b"" - - -def _decode_publish(byte0: int, body: bytes) -> tuple[str, bytes] | None: - """Extract ``(topic, payload)`` from a PUBLISH packet body.""" - if len(body) < 2: - return None - qos = (byte0 >> 1) & 3 - tlen = int.from_bytes(body[0:2], "big") - if len(body) < 2 + tlen: - return None - topic = body[2 : 2 + tlen].decode("utf-8", "replace") - off = 2 + tlen + (2 if qos > 0 else 0) - return topic, body[off:] - - async def _handle_publish(topic: str, payload: bytes) -> None: # topic: vessels-v2//location parts = topic.split("/") diff --git a/apps/api/app/foundry/connections.py b/apps/api/app/foundry/connections.py new file mode 100644 index 00000000..2cd70262 --- /dev/null +++ b/apps/api/app/foundry/connections.py @@ -0,0 +1,401 @@ +"""Operator-configured sources: an MQTT topic, a Kafka topic, a SQL query. + +Everything this platform ingests today is a source someone wrote code for. A +connection is the other half: the operator points it at THEIR broker or THEIR +database and it lands in a Foundry dataset, after which the ordinary version + +binding machinery carries it into the ontology. Nothing downstream needs to know +where a row came from. + +Three kinds, and the reason each is shaped the way it is: + +``mqtt`` Dependency-free. The MQTT 3.1.1 codec already existed for one + hard-coded broker; ``app/mqtt_client.py`` is that codec with the + broker taken out. +``kafka`` Needs ``aiokafka``, which is an OPTIONAL extra. Absent, the kind + reports itself unavailable and the app still boots — keyless boot is + a product requirement, not a dev convenience, and a deployment that + does not use Kafka must not be made to install a Kafka client. +``sql`` Needs ``sqlalchemy`` (Core only, no ORM), same optional treatment. + **The connection stores the NAME of an environment variable holding + the DSN, never the DSN.** Credentials stay in the process + environment, out of foundry.db, out of API responses, out of logs and + out of a backup of either. + +Batching, not row-at-a-time: a Foundry version is an immutable snapshot, so +writing one per message would turn a busy topic into a million versions. Rows +accumulate and flush on whichever comes first, a row count or a deadline. + +The supervisor follows the same rule the sidecars learned the hard way +(``apps/api/CLAUDE.md``): reconcile on a loop, not once at boot, or a connection +that dies at 03:00 stays dead until the next restart. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import os +import re +import time +from typing import Any + +from app.config import get_settings +from app.foundry import binding as binding_mod +from app.foundry.store import FoundryStore +from app.keys import UserCtx + +log = logging.getLogger("app.foundry.connections") + +KINDS: tuple[str, ...] = ("mqtt", "kafka", "sql") + +# A connection writes on behalf of the deployment, not a signed-in analyst. +_LOCAL_CTX = UserCtx(user_id="local", token="") + +# Flush thresholds. 500 rows keeps a version a reasonable size; 10 s keeps a +# quiet topic from sitting unwritten for minutes. +_BATCH_ROWS = 500 +_BATCH_AGE_S = 10.0 + +# How often the supervisor reconciles running tasks against the table. +RECONCILE_EVERY_S = 20.0 + +# Reconnect backoff for the streaming kinds, doubling to a ceiling. +_BACKOFF_START_S = 2.0 +_BACKOFF_MAX_S = 300.0 + +# An env var name, and nothing that could be a DSN typed into the wrong box. +_ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]{0,63}$") + + +def valid_dsn_env(name: str) -> bool: + """True for something that is an environment-variable NAME. + + Deliberately strict rather than clever: anything with a scheme, a slash, a + space or lower case is far more likely to be a connection string pasted + into the wrong field than an unusual variable name, and the cost of being + wrong in that direction is a password in the database. + """ + return bool(_ENV_NAME_RE.match(name)) + + +# ── optional dependencies ───────────────────────────────────────────────────── + + +def _probe(module: str) -> str | None: + """None when importable, else a sentence naming what to install.""" + try: + __import__(module) + except Exception: # noqa: BLE001 - a broken install is also unavailable + return f"unavailable: pip install {module}" + return None + + +def availability() -> dict[str, dict[str, Any]]: + """Which connection kinds this deployment can actually run. + + Reported rather than assumed, so the UI can grey out a kind instead of + letting an operator configure one that will only fail at run time. + """ + kafka = _probe("aiokafka") + sql = _probe("sqlalchemy") + return { + "mqtt": {"available": True, "detail": "built in"}, + "kafka": {"available": kafka is None, "detail": kafka or "aiokafka"}, + "sql": {"available": sql is None, "detail": sql or "sqlalchemy"}, + } + + +# ── batching ────────────────────────────────────────────────────────────────── + + +class _Batch: + """Rows on their way to one dataset, flushed by count or by age.""" + + def __init__(self, store: FoundryStore, conn: dict[str, Any]) -> None: + self._store = store + self._conn = conn + self._rows: list[dict[str, Any]] = [] + self._opened = time.monotonic() + + @property + def due(self) -> bool: + return bool(self._rows) and ( + len(self._rows) >= _BATCH_ROWS + or time.monotonic() - self._opened >= _BATCH_AGE_S + ) + + def add(self, row: dict[str, Any]) -> None: + self._rows.append(row) + + async def flush(self) -> int: + if not self._rows: + return 0 + rows, self._rows = self._rows, [] + self._opened = time.monotonic() + await self._store.append_version(self._conn["dataset_id"], rows) + await binding_mod.auto_sync_dataset( + self._store, self._conn["dataset_id"], _LOCAL_CTX + ) + await self._store.mark_connection( + self._conn["id"], ok=True, rows_added=len(rows) + ) + return len(rows) + + +def message_row(topic: str, payload: bytes) -> dict[str, Any]: + """One broker message as a dataset row. + + A JSON object becomes the row itself, which is what makes a binding work + without a transform in between. Anything else is kept verbatim under + ``payload`` rather than dropped, because a message this code could not read + is exactly the one an operator needs to see to fix their topic. + """ + text = payload.decode("utf-8", errors="replace") + row: dict[str, Any] = {} + try: + parsed = json.loads(text) + except (json.JSONDecodeError, ValueError): + parsed = None + if isinstance(parsed, dict): + row.update(parsed) + else: + row["payload"] = text + row.setdefault("_topic", topic) + return row + + +# ── per-kind runners ────────────────────────────────────────────────────────── + + +async def _run_mqtt(store: FoundryStore, conn: dict[str, Any]) -> None: + from app import mqtt_client + + cfg = conn["config"] + url = str(cfg.get("url") or "") + topic = str(cfg.get("topic") or "") + if not url or not topic: + raise ValueError("an mqtt connection needs a url and a topic") + batch = _Batch(store, conn) + async for msg_topic, payload in mqtt_client.subscribe( + url, topic, client_id=str(cfg.get("client_id") or "osint-geoint") + ): + batch.add(message_row(msg_topic, payload)) + if batch.due: + await batch.flush() + + +async def _run_kafka(store: FoundryStore, conn: dict[str, Any]) -> None: + from aiokafka import AIOKafkaConsumer # noqa: PLC0415 - optional dependency + + cfg = conn["config"] + topic = str(cfg.get("topic") or "") + servers = str(cfg.get("bootstrap_servers") or "") + if not topic or not servers: + raise ValueError("a kafka connection needs bootstrap_servers and a topic") + consumer = AIOKafkaConsumer( + topic, + bootstrap_servers=servers, + group_id=str(cfg.get("group_id") or "osint-geoint"), + # Only what arrives from now on: a connection is a live feed, and + # replaying a retained topic from the beginning would write a version + # per 500 messages of history nobody asked for. + auto_offset_reset=str(cfg.get("auto_offset_reset") or "latest"), + enable_auto_commit=True, + ) + await consumer.start() + batch = _Batch(store, conn) + try: + while True: + # A timed poll rather than `async for`, so an idle topic still lets + # the age-based flush fire. + got = await consumer.getmany(timeout_ms=1000) + for _tp, messages in got.items(): + for m in messages: + batch.add(message_row(getattr(m, "topic", topic), m.value or b"")) + if batch.due: + await batch.flush() + finally: + with contextlib.suppress(Exception): + await consumer.stop() + + +def _resolve_dsn(cfg: dict[str, Any]) -> str: + """The DSN behind the configured environment-variable NAME. + + The name, not the value, is what a connection row is allowed to hold: a + dump of foundry.db, an API response listing connections, and a log line are + all places a DSN with a password in it must never reach. + """ + env_name = str(cfg.get("dsn_env") or "") + if not _ENV_NAME_RE.match(env_name): + raise ValueError( + "dsn_env must be the NAME of an environment variable holding the " + "connection string (upper case, e.g. OSINT_SQL_DSN_WAREHOUSE), " + "never the connection string itself" + ) + dsn = os.environ.get(env_name) + if not dsn: + raise ValueError(f"environment variable {env_name} is not set") + return dsn + + +async def _run_sql(store: FoundryStore, conn: dict[str, Any]) -> None: + import sqlalchemy # noqa: PLC0415 - optional dependency + + cfg = conn["config"] + query = str(cfg.get("query") or "") + if not query.strip(): + raise ValueError("a sql connection needs a query") + interval = max(30.0, float(cfg.get("interval_s") or 300)) + dsn = _resolve_dsn(cfg) + + def _pull() -> list[dict[str, Any]]: + # Core, not the ORM, and a fresh engine per cycle: a poll every few + # minutes does not justify holding a pool open against someone else's + # database between runs. + engine = sqlalchemy.create_engine(dsn) + try: + with engine.connect() as c: + result = c.exec_driver_sql(query) + return [dict(r) for r in result.mappings()] + finally: + engine.dispose() + + while True: + rows = await asyncio.get_running_loop().run_in_executor(None, _pull) + if rows: + # A SQL pull is a whole answer, so it is one version, not a batch. + await store.append_version(conn["dataset_id"], rows) + await binding_mod.auto_sync_dataset( + store, conn["dataset_id"], _LOCAL_CTX + ) + await store.mark_connection(conn["id"], ok=True, rows_added=len(rows)) + await asyncio.sleep(interval) + + +_RUNNERS = {"mqtt": _run_mqtt, "kafka": _run_kafka, "sql": _run_sql} + + +def _scrub(text: str, secret: str | None) -> str: + return text.replace(secret, "***") if secret else text + + +async def _run_forever(conn: dict[str, Any]) -> None: + """One connection, restarted with backoff until it is disabled or removed. + + Errors are recorded on the row, not raised: a broker that is down is an + operator's problem to see in the UI, not a reason to take a task out of the + supervisor's hands. + """ + store = FoundryStore(get_settings()) + runner = _RUNNERS[conn["kind"]] + backoff = _BACKOFF_START_S + while True: + started = time.monotonic() + try: + await runner(store, conn) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - every failure is reportable + # Never let a driver's exception carry the DSN into the database. + secret = os.environ.get(str(conn["config"].get("dsn_env") or "")) or None + detail = _scrub(f"{type(exc).__name__}: {exc}", secret) + log.warning("connection %s failed: %s", conn["name"], detail) + with contextlib.suppress(Exception): + await store.mark_connection(conn["id"], ok=False, error=detail) + # A session that ran for a while was real; only a fast-failing one keeps + # doubling, so a broker outage backs off to minutes instead of hammering. + if time.monotonic() - started > 60.0: + backoff = _BACKOFF_START_S + await asyncio.sleep(backoff) + backoff = min(backoff * 2, _BACKOFF_MAX_S) + + +# ── supervision ─────────────────────────────────────────────────────────────── + +_tasks: dict[str, asyncio.Task[None]] = {} +_fingerprints: dict[str, str] = {} +_supervisor: asyncio.Task[None] | None = None + + +def _fingerprint(conn: dict[str, Any]) -> str: + return json.dumps( + [conn["kind"], conn["dataset_id"], conn["config"]], sort_keys=True + ) + + +async def _cancel(conn_id: str) -> None: + task = _tasks.pop(conn_id, None) + _fingerprints.pop(conn_id, None) + if task is None: + return + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + + +async def reconcile() -> None: + """Make the running tasks match the enabled rows. + + Also restarts a connection whose config changed, which is the only way an + edit in the UI takes effect without a reboot. + """ + store = FoundryStore(get_settings()) + rows = await store.list_connections() + wanted = {c["id"]: c for c in rows if c["enabled"] and c["kind"] in _RUNNERS} + + for conn_id in list(_tasks): + conn = wanted.get(conn_id) + if conn is None or _fingerprints.get(conn_id) != _fingerprint(conn): + await _cancel(conn_id) + elif _tasks[conn_id].done(): + # _run_forever only returns if it was cancelled; a finished task is + # a crash in the supervision layer itself, so restart it. + _tasks.pop(conn_id, None) + _fingerprints.pop(conn_id, None) + + availability_now = availability() + for conn_id, conn in wanted.items(): + if conn_id in _tasks: + continue + if not availability_now[conn["kind"]]["available"]: + continue + _tasks[conn_id] = asyncio.create_task( + _run_forever(conn), name=f"connection:{conn['name']}" + ) + _fingerprints[conn_id] = _fingerprint(conn) + + +async def supervise() -> None: + while True: + try: + await reconcile() + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + log.warning("connection reconcile failed", exc_info=True) + await asyncio.sleep(RECONCILE_EVERY_S) + + +async def start() -> None: + global _supervisor + if _supervisor is None or _supervisor.done(): + _supervisor = asyncio.create_task(supervise(), name="connections-supervisor") + + +async def stop() -> None: + global _supervisor + if _supervisor is not None: + _supervisor.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await _supervisor + _supervisor = None + for conn_id in list(_tasks): + await _cancel(conn_id) + + +def running_ids() -> list[str]: + """Connection ids with a live task. Used by the routes to report state.""" + return [cid for cid, t in _tasks.items() if not t.done()] diff --git a/apps/api/app/foundry/ingest.py b/apps/api/app/foundry/ingest.py index f51a4151..a2eb49c9 100644 --- a/apps/api/app/foundry/ingest.py +++ b/apps/api/app/foundry/ingest.py @@ -1,6 +1,7 @@ -"""Dataset ingest: CSV / JSON / NDJSON parsing, type inference, caps. +"""Dataset ingest: CSV / JSON / NDJSON / GeoJSON / KML parsing, type inference, caps. -Pure stdlib (``csv``, ``json``) — no pandas. Type inference casts CSV's +Pure stdlib (``csv``, ``json``, ``xml.etree``, ``zipfile``) — no pandas, no +geopandas. Type inference casts CSV's all-string cells to ``int | float | bool | str`` (JSON/NDJSON already carry typed values from ``json.loads``); the per-column ``schema`` is the union of observed value types, matching ``docs/foundry-plan.md``. @@ -170,6 +171,223 @@ def parse_ndjson(text: str) -> list[dict[str, Any]]: return rows +# ── geospatial files ────────────────────────────────────────────────────────── +# A dataset is rows, so a geospatial file has to become rows: one row per +# feature, its own properties at the top level, plus the coordinate pair and the +# raw geometry. That is enough for foundry/geo.py's lat/lon sniffer to serve the +# file straight back out of GET /api/foundry/datasets/{id}/geo, and enough for a +# binding to mint ontology objects from it. +# +# Stdlib only: json for GeoJSON, xml.etree for KML, zipfile for KMZ. Shapefile +# and LAS/LAZ are deliberately absent — both need a real dependency and neither +# has asked for itself yet. + +_GEOM_KEY = "geometry" +_GEOM_TYPE_KEY = "geometry_type" + + +def _flatten_coords(node: Any, out: list[tuple[float, float]]) -> None: + """Every [lon, lat] pair anywhere in a GeoJSON coordinates array. + + Recursive because the nesting depth is the geometry type: Point is one + pair, Polygon is rings of pairs, MultiPolygon one deeper again. + """ + if not isinstance(node, (list, tuple)): + return + if ( + len(node) >= 2 + and isinstance(node[0], (int, float)) + and isinstance(node[1], (int, float)) + and not isinstance(node[0], bool) + and not isinstance(node[1], bool) + ): + out.append((float(node[0]), float(node[1]))) + return + for child in node: + _flatten_coords(child, out) + + +def _representative_point(geometry: Any) -> tuple[float | None, float | None]: + """One (lon, lat) for a feature: the point itself, or the centre of the + bounding box for anything with extent. + + The bbox centre, not the centroid — a centroid needs the geometry's area and + can land outside a concave shape, and this value exists to put a pin on a + map, not to do geometry. + """ + if not isinstance(geometry, dict): + return None, None + pairs: list[tuple[float, float]] = [] + if geometry.get("type") == "GeometryCollection": + for g in geometry.get("geometries") or []: + _flatten_coords((g or {}).get("coordinates"), pairs) + else: + _flatten_coords(geometry.get("coordinates"), pairs) + if not pairs: + return None, None + lons = [p[0] for p in pairs] + lats = [p[1] for p in pairs] + return (min(lons) + max(lons)) / 2, (min(lats) + max(lats)) / 2 + + +def _geo_row(properties: Any, geometry: Any) -> dict[str, Any]: + """One feature as a row. The feature's own properties keep their names and + their values; ``lat``/``lon`` are only filled in when the feature did not + already carry columns by those names, because a file that states its own + coordinates knows better than a derived bbox centre.""" + row: dict[str, Any] = dict(properties) if isinstance(properties, dict) else {} + lon, lat = _representative_point(geometry) + if lat is not None and "lat" not in row: + row["lat"] = lat + if lon is not None and "lon" not in row: + row["lon"] = lon + if isinstance(geometry, dict): + row.setdefault(_GEOM_TYPE_KEY, geometry.get("type")) + # The geometry travels as a JSON string: a dataset cell is a scalar, and + # keeping the original means a later transform can still read the shape. + row.setdefault(_GEOM_KEY, json.dumps(geometry, separators=(",", ":"))) + return row + + +def parse_geojson(text: str) -> list[dict[str, Any]]: + """A FeatureCollection, a bare Feature, or a bare geometry, as rows.""" + data = json.loads(text) + if not isinstance(data, dict): + raise FoundryError(422, "GeoJSON upload must be an object") + kind = data.get("type") + if kind == "FeatureCollection": + features = data.get("features") + if not isinstance(features, list): + raise FoundryError(422, "FeatureCollection has no features array") + return [ + _geo_row((f or {}).get("properties"), (f or {}).get("geometry")) + for f in features + if isinstance(f, dict) + ] + if kind == "Feature": + return [_geo_row(data.get("properties"), data.get("geometry"))] + if kind: + return [_geo_row({}, data)] + raise FoundryError(422, "GeoJSON upload has no type") + + +def _local(tag: str) -> str: + """``{http://www.opengis.net/kml/2.2}Placemark`` → ``Placemark``. + + KML in the wild carries 2.2, 2.1, no namespace at all, and Google's + extension namespace side by side, so matching on the local name is the only + thing that reads every file the same way. + """ + return tag.rpartition("}")[2] + + +def _kml_coords(placemark: Any) -> Any: + """The placemark's geometry as a GeoJSON-ish dict, or None. + + KML writes ``lon,lat[,alt]`` tuples separated by whitespace. The type is + read from the element that holds them, so a Polygon keeps its rings shape + rather than collapsing to a bag of points. A MultiGeometry placemark yields + its FIRST geometry: a dataset row holds one shape, and picking the first in + document order is at least deterministic. + """ + kml_to_geojson = { + "Point": "Point", + "LineString": "LineString", + "LinearRing": "LineString", + "Polygon": "Polygon", + } + for el in placemark.iter(): + name = _local(el.tag) + if name != "coordinates" or not (el.text or "").strip(): + continue + pairs: list[list[float]] = [] + for chunk in (el.text or "").split(): + parts = chunk.split(",") + if len(parts) < 2: + continue + try: + pairs.append([float(parts[0]), float(parts[1])]) + except ValueError: + continue + if not pairs: + continue + # The nearest enclosing geometry element decides the shape. + holder = "Point" + for anc in placemark.iter(): + if _local(anc.tag) in kml_to_geojson and el in list(anc.iter()): + holder = _local(anc.tag) + break + gtype = kml_to_geojson.get(holder, "Point") + if gtype == "Point": + return {"type": "Point", "coordinates": pairs[0]} + if gtype == "Polygon": + return {"type": "Polygon", "coordinates": [pairs]} + return {"type": "LineString", "coordinates": pairs} + return None + + +def parse_kml(text: str) -> list[dict[str, Any]]: + """Every Placemark as a row: name, description, its ExtendedData fields, and + its geometry.""" + from xml.etree import ElementTree + + try: + root = ElementTree.fromstring(text) + except ElementTree.ParseError as exc: + raise FoundryError(422, f"could not parse KML: {exc}") from exc + rows: list[dict[str, Any]] = [] + for pm in root.iter(): + if _local(pm.tag) != "Placemark": + continue + props: dict[str, Any] = {} + for child in pm: + name = _local(child.tag) + if name in ("name", "description", "styleUrl", "address") and child.text: + props[name] = child.text.strip() + # 123 + # and the SimpleData variant schemas use. + for el in pm.iter(): + name = _local(el.tag) + key = el.get("name") + if not key: + continue + if name == "Data": + value = next( + (v.text for v in el if _local(v.tag) == "value"), None + ) + elif name == "SimpleData": + value = el.text + else: + continue + if value is not None: + props[key] = _cast_scalar(value) + rows.append(_geo_row(props, _kml_coords(pm))) + return rows + + +def parse_kmz(content: bytes) -> list[dict[str, Any]]: + """A KMZ is a zip whose payload is a KML, conventionally ``doc.kml``.""" + import zipfile + + try: + with zipfile.ZipFile(io.BytesIO(content)) as zf: + names = [n for n in zf.namelist() if n.lower().endswith(".kml")] + if not names: + raise FoundryError(422, "KMZ archive contains no .kml file") + # doc.kml is the convention; otherwise take the first, in archive + # order, so the choice is at least deterministic. + name = next((n for n in names if n.lower().endswith("doc.kml")), names[0]) + with zf.open(name) as fh: + payload = fh.read(MAX_UPLOAD_BYTES + 1) + except zipfile.BadZipFile as exc: + raise FoundryError(422, f"could not read KMZ: {exc}") from exc + if len(payload) > MAX_UPLOAD_BYTES: + raise FoundryError( + 413, f"KMZ payload too large: > {MAX_UPLOAD_BYTES} bytes uncompressed" + ) + return parse_kml(payload.decode("utf-8", errors="replace")) + + def _value_type(v: Any) -> str: if isinstance(v, bool): return "bool" @@ -211,6 +429,29 @@ def infer_schema(rows: list[dict[str, Any]]) -> list[dict[str, str]]: return schema +_GEOJSON_TYPES = { + "FeatureCollection", + "Feature", + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection", +} + + +def _declares_geojson(text: str) -> bool: + """True when a ``.json`` body is really GeoJSON. Parse failures answer False + so the JSON-array path keeps ownership of the error message.""" + try: + data = json.loads(text) + except (json.JSONDecodeError, ValueError): + return False + return isinstance(data, dict) and data.get("type") in _GEOJSON_TYPES + + def parse_upload( filename: str, content: bytes ) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: @@ -218,21 +459,39 @@ def parse_upload( Enforces the 25 MB size cap and the 200k row cap (413 / 422 respectively — the route layer maps ``FoundryError.status_code``). Format is chosen by - extension: ``.csv`` → CSV, ``.ndjson``/``.jsonl`` → NDJSON, else JSON array. + extension: ``.csv`` → CSV, ``.ndjson``/``.jsonl`` → NDJSON, + ``.geojson`` → GeoJSON features, ``.kml``/``.kmz`` → KML placemarks, + ``.json`` → GeoJSON if it declares a GeoJSON type, else a JSON array. """ if len(content) > MAX_UPLOAD_BYTES: raise FoundryError( 413, f"upload too large: {len(content)} bytes > {MAX_UPLOAD_BYTES}" ) - text = content.decode("utf-8", errors="replace") name = filename.lower() + # KMZ is a zip, so it is the one format that must not be decoded first. + if name.endswith(".kmz"): + rows = parse_kmz(content) + if len(rows) > MAX_ROWS_PER_DATASET: + raise FoundryError( + 422, f"row cap exceeded: {len(rows)} > {MAX_ROWS_PER_DATASET}" + ) + return rows, infer_schema(rows) + text = content.decode("utf-8", errors="replace") try: if name.endswith(".csv"): rows = parse_csv(text) elif name.endswith(".ndjson") or name.endswith(".jsonl"): rows = parse_ndjson(text) + elif name.endswith(".geojson"): + rows = parse_geojson(text) + elif name.endswith(".kml"): + rows = parse_kml(text) elif name.endswith(".json"): - rows = parse_json_array(text) + # A .geojson renamed .json is common enough that the extension is + # not worth trusting over the file's own declared type. + rows = ( + parse_geojson(text) if _declares_geojson(text) else parse_json_array(text) + ) else: # Fall back to sniffing: valid JSON array first, else CSV. stripped = text.lstrip() diff --git a/apps/api/app/foundry/store.py b/apps/api/app/foundry/store.py index a442e9ba..50548a9e 100644 --- a/apps/api/app/foundry/store.py +++ b/apps/api/app/foundry/store.py @@ -14,7 +14,9 @@ import asyncio import calendar +import hashlib import json +import secrets import sqlite3 import time import uuid @@ -58,7 +60,8 @@ def _resolved_db_path(settings: Settings | None = None) -> str: id TEXT PRIMARY KEY, name TEXT UNIQUE NOT NULL, description TEXT DEFAULT '', kind TEXT NOT NULL DEFAULT 'raw', schema_json TEXT NOT NULL DEFAULT '[]', - created_at TEXT NOT NULL, updated_at TEXT NOT NULL + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + ingest_token_sha256 TEXT ); CREATE TABLE IF NOT EXISTS versions ( id INTEGER PRIMARY KEY, dataset_id TEXT NOT NULL REFERENCES datasets(id), @@ -124,6 +127,13 @@ def _resolved_db_path(settings: Settings | None = None) -> str: severity TEXT NOT NULL DEFAULT 'medium', enabled INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); +CREATE TABLE IF NOT EXISTS connections ( + id TEXT PRIMARY KEY, name TEXT UNIQUE NOT NULL, kind TEXT NOT NULL, + dataset_id TEXT NOT NULL, config_json TEXT NOT NULL DEFAULT '{}', + enabled INTEGER NOT NULL DEFAULT 1, + last_ok TEXT, last_error TEXT, rows_total INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL +); CREATE TABLE IF NOT EXISTS monitor_events ( id INTEGER PRIMARY KEY, monitor_id TEXT NOT NULL, at TEXT NOT NULL, kind TEXT NOT NULL, summary TEXT NOT NULL, detail_json TEXT NOT NULL DEFAULT '{}' @@ -152,6 +162,12 @@ def _ensure_migrations(con: sqlite3.Connection) -> None: binding_cols = {r[1] for r in con.execute("PRAGMA table_info(bindings)").fetchall()} if "resolve" not in binding_cols: con.execute("ALTER TABLE bindings ADD COLUMN resolve INTEGER NOT NULL DEFAULT 0") + ds_cols = {r[1] for r in con.execute("PRAGMA table_info(datasets)").fetchall()} + if "ingest_token_sha256" not in ds_cols: + # The HASH, never the token: this column arms an endpoint an + # unauthenticated stranger can reach, so a copy of foundry.db must not + # be a copy of the credential. + con.execute("ALTER TABLE datasets ADD COLUMN ingest_token_sha256 TEXT") def _connect(settings: Settings | None = None) -> sqlite3.Connection: @@ -176,6 +192,10 @@ def new_id(prefix: str) -> str: return f"{prefix}_{uuid.uuid4().hex[:12]}" +def _token_hash(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + class FoundryError(Exception): """Raised for store-level failures the route layer maps to HTTP errors.""" @@ -298,6 +318,68 @@ def _sync() -> dict[str, Any] | None: return await self._run(_sync) + # ---- ingest tokens ---------------------------------------------------- + # A dataset with a token has an inbound push endpoint; one without does not + # exist as far as POST /api/ingest is concerned. Only the sha256 is stored, + # so the plaintext exists exactly once, in the response that mints it. + + async def mint_ingest_token(self, dataset_id: str) -> str | None: + """Arm (or re-arm) the dataset's push endpoint. Returns the plaintext + token, or None if the dataset does not exist. Re-minting invalidates the + previous token, which is the revoke-and-rotate path.""" + token = secrets.token_urlsafe(32) + + def _sync() -> str | None: + con = _connect(self.s) + try: + cur = con.execute( + "UPDATE datasets SET ingest_token_sha256=?, updated_at=?" + " WHERE id=?", + (_token_hash(token), _now_iso(), dataset_id), + ) + con.commit() + return token if cur.rowcount else None + finally: + con.close() + + return await self._run(_sync) + + async def clear_ingest_token(self, dataset_id: str) -> bool: + def _sync() -> bool: + con = _connect(self.s) + try: + cur = con.execute( + "UPDATE datasets SET ingest_token_sha256=NULL, updated_at=?" + " WHERE id=?", + (_now_iso(), dataset_id), + ) + con.commit() + return bool(cur.rowcount) + finally: + con.close() + + return await self._run(_sync) + + async def ingest_token_matches(self, dataset_id: str, token: str) -> bool | None: + """True/False when the dataset has a token, None when it has none (or + does not exist) — the caller turns None into the same 404 for both, so + the endpoint cannot be used to enumerate dataset ids.""" + + def _sync() -> bool | None: + con = _connect(self.s) + try: + row = con.execute( + "SELECT ingest_token_sha256 FROM datasets WHERE id=?", + (dataset_id,), + ).fetchone() + finally: + con.close() + if row is None or not row[0]: + return None + return secrets.compare_digest(row[0], _token_hash(token)) + + return await self._run(_sync) + async def get_dataset_by_name(self, name: str) -> dict[str, Any] | None: def _sync() -> dict[str, Any] | None: con = _connect(self.s) @@ -1351,6 +1433,169 @@ def _binding_row(self, row: tuple[Any, ...]) -> dict[str, Any]: " last_sync, last_result_json, created_at, resolve" ) + # ---- connections ------------------------------------------------------ + # A connection is a source the OPERATOR configured: an MQTT topic, a Kafka + # topic, a query against their own SQL database. The runner in + # ``foundry/connections.py`` batches whatever arrives into ``dataset_id``, + # after which the ordinary version + binding machinery takes over. + + _CONNECTION_COLS = ( + "id, name, kind, dataset_id, config_json, enabled, last_ok," + " last_error, rows_total, created_at, updated_at" + ) + + @staticmethod + def _connection_row(row: tuple[Any, ...]) -> dict[str, Any]: + return { + "id": row[0], + "name": row[1], + "kind": row[2], + "dataset_id": row[3], + "config": json.loads(row[4]), + "enabled": bool(row[5]), + "last_ok": row[6], + "last_error": row[7], + "rows_total": row[8], + "created_at": row[9], + "updated_at": row[10], + } + + async def list_connections(self) -> list[dict[str, Any]]: + def _sync() -> list[dict[str, Any]]: + con = _connect(self.s) + try: + rows = con.execute( + f"SELECT {self._CONNECTION_COLS} FROM connections" + " ORDER BY created_at DESC" + ).fetchall() + finally: + con.close() + return [self._connection_row(r) for r in rows] + + return await self._run(_sync) + + async def get_connection(self, connection_id: str) -> dict[str, Any] | None: + def _sync() -> dict[str, Any] | None: + con = _connect(self.s) + try: + row = con.execute( + f"SELECT {self._CONNECTION_COLS} FROM connections WHERE id=?", + (connection_id,), + ).fetchone() + finally: + con.close() + return self._connection_row(row) if row else None + + return await self._run(_sync) + + async def create_connection( + self, + name: str, + kind: str, + dataset_id: str, + config: dict[str, Any], + enabled: bool = True, + ) -> dict[str, Any]: + def _sync() -> dict[str, Any]: + con = _connect(self.s) + try: + if con.execute( + "SELECT id FROM connections WHERE name=?", (name,) + ).fetchone(): + raise FoundryError(409, f"connection {name!r} already exists") + cid = new_id("conn") + now = _now_iso() + con.execute( + "INSERT INTO connections (id, name, kind, dataset_id," + " config_json, enabled, created_at, updated_at)" + " VALUES (?,?,?,?,?,?,?,?)", + (cid, name, kind, dataset_id, json.dumps(config), int(enabled), now, now), + ) + con.commit() + row = con.execute( + f"SELECT {self._CONNECTION_COLS} FROM connections WHERE id=?", + (cid,), + ).fetchone() + return self._connection_row(row) + finally: + con.close() + + return await self._run(_sync) + + async def update_connection( + self, + connection_id: str, + *, + dataset_id: str, + config: dict[str, Any], + enabled: bool, + ) -> dict[str, Any] | None: + def _sync() -> dict[str, Any] | None: + con = _connect(self.s) + try: + cur = con.execute( + "UPDATE connections SET dataset_id=?, config_json=?, enabled=?," + " updated_at=? WHERE id=?", + (dataset_id, json.dumps(config), int(enabled), _now_iso(), connection_id), + ) + con.commit() + if not cur.rowcount: + return None + row = con.execute( + f"SELECT {self._CONNECTION_COLS} FROM connections WHERE id=?", + (connection_id,), + ).fetchone() + return self._connection_row(row) + finally: + con.close() + + return await self._run(_sync) + + async def delete_connection(self, connection_id: str) -> bool: + def _sync() -> bool: + con = _connect(self.s) + try: + cur = con.execute( + "DELETE FROM connections WHERE id=?", (connection_id,) + ) + con.commit() + return bool(cur.rowcount) + finally: + con.close() + + return await self._run(_sync) + + async def mark_connection( + self, + connection_id: str, + *, + ok: bool, + error: str | None = None, + rows_added: int = 0, + ) -> None: + """Record the outcome of one runner cycle. ``last_error`` is cleared on a + good cycle so the row shows the CURRENT state, not the worst one ever.""" + + def _sync() -> None: + con = _connect(self.s) + try: + if ok: + con.execute( + "UPDATE connections SET last_ok=?, last_error=NULL," + " rows_total=rows_total+? WHERE id=?", + (_now_iso(), int(rows_added), connection_id), + ) + else: + con.execute( + "UPDATE connections SET last_error=? WHERE id=?", + ((error or "")[:500], connection_id), + ) + con.commit() + finally: + con.close() + + await self._run(_sync) + async def create_binding( self, dataset_id: str, diff --git a/apps/api/app/intel/action_proposals_local.py b/apps/api/app/intel/action_proposals_local.py new file mode 100644 index 00000000..cd864b0f --- /dev/null +++ b/apps/api/app/intel/action_proposals_local.py @@ -0,0 +1,181 @@ +"""Local SQLite store for the human-in-the-loop action queue. + +A proposal is a governed write-back the agent wants to make and is waiting for +an operator to approve. It used to live in a module-level dict, which meant a +backend restart silently emptied the approval queue: the operator saw pending +work, the process bounced, and the work was gone with no record that it had +existed. "The agent re-proposes on its next run" was the stated defence, and it +only holds if the agent runs again — an operator who left three proposals open +overnight came back to none. + +Same idiom as ``action_log_local.py`` beside it: WAL SQLite under ``./data``, a +fresh connection per operation off the default executor, an +``override_db_path()`` test hook, and no ``Settings`` entry for a path nobody +has asked to relocate. + +Expiry is unchanged and is enforced on read as well as on write, so a proposal +that aged out while the process was down is never handed back as pending. +""" + +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import time +import uuid +from pathlib import Path +from typing import Any + +_DEFAULT_DB_PATH = "./data/action_proposals.db" + +# ── DB path injection (for tests) ───────────────────────────────────────────── + +_db_path_override: str | None = None + + +def override_db_path(path: str | None) -> None: + """Set a custom DB path (tests). Pass None to clear.""" + global _db_path_override + _db_path_override = path + + +def _resolved_db_path() -> str: + return _db_path_override or _DEFAULT_DB_PATH + + +# ── connection / schema ─────────────────────────────────────────────────────── + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS action_proposals ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + params TEXT NOT NULL DEFAULT '{}', + confidence REAL NOT NULL DEFAULT 0.0, + created REAL NOT NULL +); +CREATE INDEX IF NOT EXISTS ix_action_proposals_created + ON action_proposals(created); +""" + + +def _connect() -> sqlite3.Connection: + path = _resolved_db_path() + Path(path).parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(path, check_same_thread=False) + con.execute("PRAGMA journal_mode=WAL") + con.execute("PRAGMA busy_timeout=5000") + con.executescript(_SCHEMA) + con.commit() + return con + + +async def _run(fn: Any) -> Any: + return await asyncio.get_running_loop().run_in_executor(None, fn) + + +def _row(r: tuple[Any, ...]) -> dict[str, Any]: + return { + "id": r[0], + "name": r[1], + "params": json.loads(r[2]), + "confidence": r[3], + "created": r[4], + } + + +async def add( + name: str, params: dict[str, Any], confidence: float, ttl_s: float +) -> str: + """Queue a proposal and return its id, pruning anything already expired.""" + pid = uuid.uuid4().hex[:12] + now = time.time() + + def _sync() -> None: + con = _connect() + try: + con.execute( + "DELETE FROM action_proposals WHERE created < ?", (now - ttl_s,) + ) + con.execute( + "INSERT INTO action_proposals (id, name, params, confidence, created)" + " VALUES (?,?,?,?,?)", + (pid, name, json.dumps(params), float(confidence), now), + ) + con.commit() + finally: + con.close() + + await _run(_sync) + return pid + + +async def list_pending(ttl_s: float) -> list[dict[str, Any]]: + """Unexpired proposals, oldest first. + + Filters by age in the query rather than trusting a prune to have run: after + a restart nothing has pruned yet, and a proposal that expired while the + process was down must not come back looking live. + """ + + def _sync() -> list[dict[str, Any]]: + con = _connect() + try: + rows = con.execute( + "SELECT id, name, params, confidence, created FROM action_proposals" + " WHERE created >= ? ORDER BY created ASC", + (time.time() - ttl_s,), + ).fetchall() + finally: + con.close() + return [_row(r) for r in rows] + + return await _run(_sync) + + +async def take(pid: str, ttl_s: float) -> dict[str, Any] | None: + """Remove and return one unexpired proposal, or None. + + Delete-then-read in one connection so two approvals of the same proposal + cannot both execute it. + """ + + def _sync() -> dict[str, Any] | None: + con = _connect() + try: + row = con.execute( + "SELECT id, name, params, confidence, created FROM action_proposals" + " WHERE id=? AND created >= ?", + (pid, time.time() - ttl_s), + ).fetchone() + if row is None: + # Still clear an expired row of the same id so the table does + # not keep one nobody can act on. + con.execute("DELETE FROM action_proposals WHERE id=?", (pid,)) + con.commit() + return None + con.execute("DELETE FROM action_proposals WHERE id=?", (pid,)) + con.commit() + return _row(row) + finally: + con.close() + + return await _run(_sync) + + +async def prune(ttl_s: float) -> int: + """Drop expired rows; returns how many went.""" + + def _sync() -> int: + con = _connect() + try: + cur = con.execute( + "DELETE FROM action_proposals WHERE created < ?", + (time.time() - ttl_s,), + ) + con.commit() + return cur.rowcount or 0 + finally: + con.close() + + return await _run(_sync) diff --git a/apps/api/app/intel/agent.py b/apps/api/app/intel/agent.py index 7e4cb8d4..11bf0e01 100644 --- a/apps/api/app/intel/agent.py +++ b/apps/api/app/intel/agent.py @@ -900,7 +900,7 @@ def _index(brief: dict[str, Any]) -> None: _s = get_settings() confidence = float(args.pop("confidence", 0.0) or 0.0) if _s.action_approval and confidence < _s.action_auto_threshold: - pid = propose(name, args, ctx, confidence=confidence) + pid = await propose(name, args, ctx, confidence=confidence) yield { "type": "action_proposal", "step": step, "proposal_id": pid, "action": name, diff --git a/apps/api/app/intel/ontology.py b/apps/api/app/intel/ontology.py index aa76d2dd..5a1ad42b 100644 --- a/apps/api/app/intel/ontology.py +++ b/apps/api/app/intel/ontology.py @@ -31,6 +31,7 @@ from pydantic import BaseModel, Field from app.config import Settings, get_settings +from app.intel.ontology_schema import REL_TYPES from app.keys import UserCtx if TYPE_CHECKING: # runtime import lives in get_registry (module cycle) @@ -100,41 +101,12 @@ ) ) -# Link relations seeded by the first write-back actions + the fusion engine. -# Not an enum (the registry must hold an edge an analyst/agent invents), but the -# canonical verbs are documented here so callers stay consistent. -KNOWN_RELS: frozenset[str] = frozenset( - ( - "flagged", # analyst flagged this object (object → flag note) - "evidence_of", # signal/track → incident it supports - "promoted_to", # object → incident promoted from it - "nominated", # object → target_board entry - "watched_by", # object → alert_rule watching it - "operates", # operator → aircraft/vessel - "correlated", # cross-domain co-location (from the correlations index) - "member_of", # sim drone → swarm - "contains", # situation → child incident/entity/COA it aggregates - "part_of", # inverse of contains (child → situation) - # Phase 0 OSINT source expansion (docs/osint-sources-plan.md) — verbs - # for the new url/wallet/tx/file kinds and their infra/threat context. - "archived_url", # domain/url → its wayback-preserved copy - "contacted", # url/malware → ip it was observed talking to - "peers_with", # asn → peer asn (BGP peering/upstream relationship) - "tor_exit", # ip → threat node flagging it as a Tor exit relay - "listed_by", # ip/url/hash → threat feed that listed it - "distributes", # url → file it serves/hosts - "sends_to", # wallet → tx (outbound transfer) - "receives_from", # wallet → tx (inbound transfer) - "officer_of", # person → org they're an officer/director of - "sanctioned_as", # org/person → threat node (sanctions list entry) - "same_as", # object → wikidata entity bridge (entity resolution) - "posted_by", # reddit/social activity → username that posted it - # Country-OSINT catalog (docs/country-osint-spec.md). - "has_resource", # country → resource (a toolkit entry for that country) - "hosted_at", # resource → domain (the resource URL's host — bridges - # into the same domain: node the digital-OSINT investigate() enriches) - ) -) +# Link relations. Not an enum (the registry must hold an edge an analyst or an +# agent invents), and no longer a second hand-kept list: the verbs, both of each +# verb's names, and the kinds its endpoints are expected to be all live in +# ``intel/ontology_schema.py``. Keeping membership derived is why the drift that +# module's header documents cannot happen again. +KNOWN_RELS: frozenset[str] = frozenset(REL_TYPES) def kind_of(object_id: str) -> ObjectKind: diff --git a/apps/api/app/intel/ontology_local.py b/apps/api/app/intel/ontology_local.py index 10856e47..900cd50b 100644 --- a/apps/api/app/intel/ontology_local.py +++ b/apps/api/app/intel/ontology_local.py @@ -27,6 +27,7 @@ import asyncio import json import logging +import re import sqlite3 import time from pathlib import Path @@ -53,6 +54,10 @@ def override_db_path(path: str | None) -> None: """Set a custom DB path (tests). Pass None to clear.""" global _db_path_override _db_path_override = path + # Pointing at a different file invalidates the per-path "FTS is already + # backfilled" memo — a tmp path can be reused across tests with different + # contents, and a stale memo would leave the second one unsearchable. + _fts_backfilled.clear() def _resolved_db_path(settings: Settings | None = None) -> str: @@ -110,8 +115,21 @@ def _resolved_db_path(settings: Settings | None = None) -> str: ); CREATE INDEX IF NOT EXISTS ix_links_src ON links(user_id, src); CREATE INDEX IF NOT EXISTS ix_links_dst ON links(user_id, dst); +CREATE VIRTUAL TABLE IF NOT EXISTS objects_fts USING fts5( + id, kind, text, user_id UNINDEXED, tokenize='porter unicode61' +); """ +# The FTS index is maintained from Python (``_fts_write_sync`` below) rather +# than by SQLite triggers, because what belongs in it is the FLATTENED prop +# VALUES — a trigger only ever sees the raw JSON blob in the ``props`` column +# and would index its braces and quoting along with the words. +# +# One value per prop, truncated, so a single object carrying a large blob (a +# situation's node list, an evidence manifest) cannot dominate the index. +_FTS_VALUE_CHARS = 200 +_FTS_MAX_PROPS = 60 + def _connect(settings: Settings | None = None) -> sqlite3.Connection: path = _resolved_db_path(settings) @@ -133,6 +151,78 @@ def _canon(value: Any) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":")) +# ── full-text index ─────────────────────────────────────────────────────────── + + +def _fts_text(object_id: str, kind: str, props: dict[str, Any]) -> str: + """The searchable body of an object: its id, its kind, and every property + name paired with a flattened rendering of its value. + + Property NAMES are indexed too, so "callsign" finds every object that + reports one — the Explorer facet case — not just objects whose value happens + to contain the word. + """ + parts = [object_id, object_id.replace(":", " "), kind] + for i, (name, value) in enumerate(props.items()): + if i >= _FTS_MAX_PROPS: + break + parts.append(str(name)) + if value is None or isinstance(value, bool): + continue + if isinstance(value, (str, int, float)): + parts.append(str(value)[:_FTS_VALUE_CHARS]) + else: + # Lists and dicts: index the JSON with its punctuation stripped, so + # a nested name is a word rather than `["name"` . + parts.append( + re.sub(r"[^0-9A-Za-z_]+", " ", _canon(value))[:_FTS_VALUE_CHARS] + ) + return " ".join(parts) + + +def _fts_write_sync( + con: sqlite3.Connection, + user_id: str, + object_id: str, + kind: str, + props: dict[str, Any], +) -> None: + """Re-index one object. Delete-then-insert: FTS5 has no upsert, and an + external-content table would have to be kept in step with the same manual + writes anyway.""" + _fts_delete_sync(con, user_id, object_id) + con.execute( + "INSERT INTO objects_fts (id, kind, text, user_id) VALUES (?,?,?,?)", + (object_id, kind, _fts_text(object_id, kind, props), user_id), + ) + + +def _fts_delete_sync(con: sqlite3.Connection, user_id: str, object_id: str) -> None: + con.execute( + "DELETE FROM objects_fts WHERE user_id=? AND id=?", (user_id, object_id) + ) + + +def _fts_match(q: str) -> str: + """A user's words as an FTS5 prefix query, or "" when there is nothing to + search for. + + Never interpolate raw input into MATCH: FTS5 treats ``"``, ``*``, ``:``, + ``^``, ``-`` and ``NEAR`` as syntax, so a callsign like ``AAL-123`` or a + stray quote raises OperationalError instead of returning no rows. Reducing + the input to word tokens and quoting each one makes any input legal, and the + trailing ``*`` is what makes a Find panel feel like a Find panel. + """ + tokens = re.findall(r"[0-9A-Za-z_]+", q) + return " ".join(f'"{t}"*' for t in tokens) + + +# Backfill is per database path and per process: an existing deployment has +# objects that predate this index, and rebuilding them on every _connect (which +# happens once per operation) would put two COUNT(*)s in front of every write. +_fts_backfilled: set[str] = set() + + # Soft byte-cap bookkeeping (module-level, same philosophy as history.py's # hourly maintenance): re-check the file size at most once an hour or every # 500 writes, whichever comes first. @@ -227,6 +317,9 @@ def _sync() -> Object: None, {"op": "remove"}, ) + _fts_write_sync( + con, self.ctx.user_id, obj.id, obj.kind, obj.props + ) self._enforce_object_cap_sync(con, obj.id) con.commit() self._maybe_enforce_size_cap(con) @@ -276,6 +369,79 @@ def _sync() -> list[Object]: return await self._run(_sync) + async def search( + self, q: str, kinds: list[str] | None = None, limit: int = 50 + ) -> list[Object]: + """Objects matching ``q`` anywhere in their id, kind or property values, + best match first. + + Until this existed the only way to reach an object was to already know + its exact canonical id: ``get`` is id-exact and ``list_by_kind`` filters + on one props field. ``/api/search/objects`` is a different thing — it + searches the LIVE observation store, which holds only what is currently + being emitted, not what was promoted into the graph. + """ + match = _fts_match(q) + if not match: + return [] + + def _sync() -> list[Object]: + con = _connect(self.s) + try: + self._backfill_fts_sync(con) + sql = ( + "SELECT o.id, o.kind, o.props, o.classification," + " o.compartments, o.shared, o.created_at" + " FROM objects_fts JOIN objects o" + " ON o.user_id = objects_fts.user_id AND o.id = objects_fts.id" + " WHERE objects_fts MATCH ? AND objects_fts.user_id = ?" + ) + params: list[Any] = [match, self.ctx.user_id] + if kinds: + placeholders = ",".join("?" * len(kinds)) + sql += f" AND o.kind IN ({placeholders})" + params.extend(kinds) + sql += " ORDER BY bm25(objects_fts) LIMIT ?" + params.append(int(limit)) + try: + rows = con.execute(sql, params).fetchall() + except sqlite3.OperationalError: + # _fts_match is meant to make any input legal; if a query + # still reaches FTS5 as syntax, answering "no matches" beats + # a 500 on a search box. + log.warning("ontology search rejected by FTS5: %r", q) + return [] + finally: + con.close() + return [_object_from_row(r) for r in rows] + + return await self._run(_sync) + + def _backfill_fts_sync(self, con: sqlite3.Connection) -> None: + """Index objects written before this table existed. Once per DB path per + process, and only when the index is genuinely behind.""" + path = _resolved_db_path(self.s) + if path in _fts_backfilled: + return + _fts_backfilled.add(path) + (indexed,) = con.execute("SELECT COUNT(*) FROM objects_fts").fetchone() + (total,) = con.execute("SELECT COUNT(*) FROM objects").fetchone() + if indexed >= total: + return + log.info("ontology FTS backfill: %d objects", total) + rows = con.execute("SELECT user_id, id, kind, props FROM objects").fetchall() + con.execute("DELETE FROM objects_fts") + for user_id, obj_id, kind, props_json in rows: + try: + props = json.loads(props_json) + except (TypeError, ValueError): + props = {} + con.execute( + "INSERT INTO objects_fts (id, kind, text, user_id) VALUES (?,?,?,?)", + (obj_id, kind, _fts_text(obj_id, kind, props), user_id), + ) + con.commit() + async def delete(self, object_id: str) -> None: """Delete an object plus its assertions and touching links. @@ -299,6 +465,7 @@ def _sync() -> None: "DELETE FROM links WHERE user_id=? AND (src=? OR dst=?)", (self.ctx.user_id, object_id, object_id), ) + _fts_delete_sync(con, self.ctx.user_id, object_id) con.commit() finally: con.close() @@ -455,6 +622,9 @@ def _sync() -> Object: now, ), ) + _fts_write_sync( + con, self.ctx.user_id, object_id, kind_of(object_id), merged + ) self._enforce_object_cap_sync(con, object_id) con.commit() out = con.execute( diff --git a/apps/api/app/intel/ontology_schema.py b/apps/api/app/intel/ontology_schema.py new file mode 100644 index 00000000..ce25220d --- /dev/null +++ b/apps/api/app/intel/ontology_schema.py @@ -0,0 +1,284 @@ +"""The ontology's declared shape — what kinds carry, and what a relation means +read from either end. + +``intel/ontology.py`` owns the typed models; this module owns the *schema over* +them. Two tables: + + ``REL_TYPES`` — every relation verb with BOTH of its names. Gotham states a + link twice ("A employs B" / "B employed by A") because a + graph is read from whichever node you are standing on, and + until now this repo only had the forward name, so an edge + traversed backwards rendered as a verb pointing the wrong + way. Each entry also names the object kinds the endpoints + are expected to be, which is what makes a mis-wired edge + visible. + ``PROP_TYPES`` — the properties a kind is known to carry, and their type. + Seeded ONLY from props this repo demonstrably writes (the + ADS-B and AIS feature builders, ``intel/promotion.py``, + ``intel/evidence.py``); a kind with nothing verified gets no + entry rather than an invented one. + +**Everything here warns, nothing rejects.** ``ontology.py`` records the operator +decision that the registry must be able to hold an edge an analyst or an agent +invents, and ``routes/extract.py`` mints links whose ``rel`` comes out of a +language model. A validator that raised would revoke that decision and start +dropping data. ``validate_object`` / ``validate_link`` return a list of English +sentences; the caller decides whether anyone reads them. + +Import direction is one-way: ``ontology.py`` imports THIS module (for +``KNOWN_RELS``), so nothing here may import from ``ontology.py``. That is why +kinds are typed ``str`` and not ``ObjectKind``. +""" + +from __future__ import annotations + +from typing import Any, Literal, NamedTuple + +# Property value types. Deliberately coarse: this drives a facet picker and a +# binding form, not a serializer. +PropType = Literal["str", "num", "bool", "ts", "geo", "id"] + + +class RelType(NamedTuple): + """One relation verb, named from both ends. + + ``forward`` reads ``src -> dst``; ``inverse`` reads ``dst -> src``. An empty + ``src``/``dst`` means "any kind" — used where the verb genuinely spans the + graph (``same_as``, ``mentions``) rather than where nobody checked. + """ + + forward: str + inverse: str + src: frozenset[str] = frozenset() + dst: frozenset[str] = frozenset() + + +def _r( + forward: str, + inverse: str, + src: tuple[str, ...] = (), + dst: tuple[str, ...] = (), +) -> RelType: + return RelType(forward, inverse, frozenset(src), frozenset(dst)) + + +# ── relations ───────────────────────────────────────────────────────────────── +# The union of the vocabulary ontology.py documented and the verbs the code +# actually mints. Those two had drifted: thirteen relations in live use +# (`mentions` from routes/extract.py, `resolves_to` / `registered_by` / +# `has_subdomain` / `secured_by` / `indicates_threat` / `announces` / +# `runs_service` / `abuse_contact` / `has_email` / `has_account` / +# `registrant_email` from routes/osint.py, `evidence` from intel/evidence.py) +# were absent from the frozenset that claimed to list them. + +REL_TYPES: dict[str, RelType] = { + # Analyst / action write-back (intel/actions.py, intel/promotion.py). + "flagged": _r("flagged", "flag on"), + "evidence_of": _r("evidence of", "supported by", dst=("incident",)), + "promoted_to": _r("promoted to", "promoted from", dst=("incident",)), + "nominated": _r("nominated", "nomination of"), + "watched_by": _r("watched by", "watches"), + "operates": _r("operates", "operated by", ("org", "person"), ("aircraft", "vessel")), + "correlated": _r("correlated with", "correlated with"), + "member_of": _r("member of", "has member", ("sim",), ("sim",)), + # Situation composition (routes/situations.py). Both directions are stored + # as separate rels here for history; the labels agree with each other. + "contains": _r("contains", "part of"), + "part_of": _r("part of", "contains"), + "evidence": _r("evidence", "evidence for", dst=("evidence",)), + # Digital OSINT (app/osint, routes/osint.py). + "archived_url": _r("archived as", "archive of", ("domain", "url"), ("url",)), + "contacted": _r("contacted", "contacted by", dst=("ip",)), + "peers_with": _r("peers with", "peers with", ("asn",), ("asn",)), + "tor_exit": _r("flagged Tor exit", "Tor exit flag on", ("ip",), ("threat",)), + "listed_by": _r("listed by", "lists", dst=("threat",)), + "distributes": _r("distributes", "distributed by", ("url",), ("file",)), + "sends_to": _r("sends to", "received from", ("wallet",), ("tx",)), + "receives_from": _r("receives from", "sent to", ("wallet",), ("tx",)), + "officer_of": _r("officer of", "has officer", ("person",), ("org",)), + "sanctioned_as": _r("sanctioned as", "sanction on", ("org", "person"), ("threat",)), + "same_as": _r("same as", "same as"), + "posted_by": _r("posted by", "posted", dst=("username",)), + "resolves_to": _r("resolves to", "resolved from", ("domain",), ("ip",)), + "registered_by": _r("registered by", "registrant of", ("domain",), ("org",)), + "registrant_email": _r( + "registrant email", "registrant email for", ("domain",), ("email",) + ), + "has_subdomain": _r("has subdomain", "subdomain of", ("domain",), ("domain",)), + "secured_by": _r("secured by", "secures", ("domain",), ("cert",)), + "indicates_threat": _r("indicates threat to", "flagged by", ("threat",)), + "announces": _r("announces", "announced by", ("asn",), ("ip",)), + "runs_service": _r("runs service", "run by", ("ip",), ("service",)), + "abuse_contact": _r("abuse contact", "abuse contact for", ("ip",), ("email",)), + "has_email": _r("has email", "email of", ("person",), ("email",)), + "has_account": _r("has account", "account of", ("person",), ("username",)), + # Document extraction (routes/extract.py). The model may invent others. + "mentions": _r("mentions", "mentioned in"), + # Country-OSINT catalog (app/osint/country_catalog.py). + "has_resource": _r("has resource", "resource for", ("country",), ("resource",)), + "hosted_at": _r("hosted at", "hosts", ("resource",), ("domain",)), +} + + +# ── properties ──────────────────────────────────────────────────────────────── +# Each block is copied from the code that writes it, cited so the next editor +# can check rather than trust. + +PROP_TYPES: dict[str, dict[str, PropType]] = { + # routes/adsb.py::_features props dict. The EntityPanel promotes exactly + # these (`snap.properties`) into the ontology. + "aircraft": { + "icao24": "id", + "callsign": "str", + "registration": "str", + "type": "str", + "category": "str", + "on_ground": "bool", + "velocity_ms": "num", + "track_deg": "num", + "baro_alt_m": "num", + "geo_alt_m": "num", + "squawk": "str", + "emergency": "str", + "nac_p": "num", + "nic": "num", + "sil": "num", + "nac_v": "num", + "seen_pos_s": "num", + "seen_at": "ts", + "source": "str", + }, + # routes/ais.py::_normalise out dict. + "vessel": { + "mmsi": "id", + "name": "str", + "lat": "num", + "lon": "num", + "sog": "num", + "cog": "num", + "heading": "num", + "shipType": "str", + "msgType": "str", + "t": "ts", + }, + # intel/promotion.py::promote_incident assert_props. + "incident": { + "threat_level": "str", + "score": "num", + "domains": "str", + "narrative": "str", + "centroid": "geo", + }, + # intel/evidence.py capture props. + "evidence": { + "sha256": "id", + "captured_at": "ts", + "captured_by": "str", + "capture_method": "str", + "filename": "str", + "final_url": "str", + "hash_algorithm": "str", + "blob_present": "bool", + }, +} + + +# ── validation (warnings only) ──────────────────────────────────────────────── + + +def validate_object(kind: str, props: dict[str, Any]) -> list[str]: + """Sentences describing how ``props`` departs from what ``kind`` declares. + + An undeclared kind, or an undeclared property on a declared kind, is not a + warning: kinds accrete faster than this table does, and a false warning on + every OSINT mint would train everyone to ignore the field. Only a property + whose declared type the value contradicts is worth saying out loud. + """ + declared = PROP_TYPES.get(kind) + if not declared: + return [] + out: list[str] = [] + for name, value in props.items(): + want = declared.get(name) + if want is None or value is None: + continue + if not _matches(want, value): + out.append( + f"{kind}.{name} is declared {want} but got " + f"{type(value).__name__} ({value!r:.40})" + ) + return out + + +def validate_link(rel: str, src_kind: str, dst_kind: str) -> list[str]: + """Sentences describing how a link departs from its declared relation. + + An unknown ``rel`` warns once (it is probably a typo, or a model-invented + verb worth promoting into ``REL_TYPES``) but the link is still writable. + """ + rt = REL_TYPES.get(rel) + if rt is None: + return [f"{rel!r} is not a declared relation"] + out: list[str] = [] + if rt.src and src_kind not in rt.src: + out.append( + f"{rel!r} expects a source of {_join(rt.src)} but got {src_kind!r}" + ) + if rt.dst and dst_kind not in rt.dst: + out.append( + f"{rel!r} expects a target of {_join(rt.dst)} but got {dst_kind!r}" + ) + return out + + +def label_for(rel: str, *, reverse: bool = False) -> str: + """The human label for ``rel``, read forwards or from the target's end. + + Falls back to the raw verb with underscores opened up, so an invented + relation still renders as words rather than as `snake_case`. + """ + rt = REL_TYPES.get(rel) + if rt is None: + return rel.replace("_", " ") + return rt.inverse if reverse else rt.forward + + +def schema_payload() -> dict[str, Any]: + """The whole declared schema, shaped for ``GET /api/ontology/schema``.""" + return { + "relations": { + rel: { + "forward": rt.forward, + "inverse": rt.inverse, + "src_kinds": sorted(rt.src), + "dst_kinds": sorted(rt.dst), + } + for rel, rt in sorted(REL_TYPES.items()) + }, + "kinds": { + kind: dict(sorted(props.items())) + for kind, props in sorted(PROP_TYPES.items()) + }, + } + + +_NUMERIC = (int, float) + + +def _matches(want: PropType, value: Any) -> bool: + # bool is an int subclass, so it has to be excluded from the numeric check + # or every True would pass as a number. + if want == "bool": + return isinstance(value, bool) + if want in ("num", "ts"): + return isinstance(value, _NUMERIC) and not isinstance(value, bool) + if want in ("str", "id"): + # ids arrive as ints from AIS (mmsi) and as strings from ADS-B (icao24). + return isinstance(value, (str, int)) and not isinstance(value, bool) + if want == "geo": + return isinstance(value, (list, tuple, dict)) + return True + + +def _join(kinds: frozenset[str]) -> str: + return " or ".join(sorted(kinds)) diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 3a042136..fd9611c0 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -104,6 +104,7 @@ def _get_libc() -> ctypes.CDLL | None: from app.routes import history as history_routes from app.routes import imagery as imagery_routes from app.routes import infra as infra_routes +from app.routes import ingest as ingest_routes from app.routes import instability as instability_routes from app.routes import intel as intel_routes from app.routes import jamming as jamming_routes @@ -385,6 +386,13 @@ def _rss_mb() -> int: from app.foundry import scheduler as foundry_scheduler # noqa: PLC0415 await foundry_scheduler.start() + # Operator-configured sources (MQTT / Kafka / SQL). SUPERVISED, not + # started once: the reconcile loop restarts a connection that dies + # later and picks up an edit made in the UI. Idles cheaply with no + # connections configured. + from app.foundry import connections as foundry_connections # noqa: PLC0415 + + await foundry_connections.start() # Workflows interval schedules: re-run a workflow on its # configured cadence. Idles cheaply with no schedules registered. from app.workflows import scheduler as workflows_scheduler # noqa: PLC0415 @@ -481,6 +489,9 @@ async def _warm_maritime() -> None: from app.foundry import scheduler as foundry_scheduler # noqa: PLC0415 await foundry_scheduler.stop() + from app.foundry import connections as foundry_connections # noqa: PLC0415 + + await foundry_connections.stop() from app.workflows import scheduler as workflows_scheduler # noqa: PLC0415 await workflows_scheduler.stop() @@ -693,6 +704,9 @@ def create_app() -> FastAPI: # Foundry substrate: BYO-data datasets/transforms/builds/ontology bindings # (docs/foundry-plan.md). Local SQLite, keyless. app.include_router(foundry_routes.router) + # Inbound push: the ONE route an unauthenticated sender can write through, + # gated by a per-dataset token rather than a session (app/routes/ingest.py). + app.include_router(ingest_routes.router) # Workflows: user-authored DAG pipelines over live platform data # (docs/dashboard-workflows-plan.md). Local SQLite, keyless. app.include_router(workflows_routes.router) diff --git a/apps/api/app/mqtt_client.py b/apps/api/app/mqtt_client.py new file mode 100644 index 00000000..9f94ef81 --- /dev/null +++ b/apps/api/app/mqtt_client.py @@ -0,0 +1,233 @@ +"""A minimal MQTT 3.1.1 subscriber, over WebSocket or plain TCP. + +The wire codec here was extracted verbatim from ``app/ais_keyless.py``, where it +was written to reach one hard-coded broker (Digitraffic). Nothing about encoding +a CONNECT packet is Digitraffic-specific, so a user-configurable MQTT connection +needs no new dependency and no second implementation — ``ais_keyless`` imports +these same functions and its existing guard test still exercises them, which is +what makes the move provably behaviour-preserving. + +Only what a consumer needs: CONNECT, SUBSCRIBE, PINGREQ keepalive, and PUBLISH +decoding at QoS 0/1/2. No publishing, no session resumption, no MQTT 5. If a +broker ever needs those, that is the moment to weigh a real dependency. + +Backoff and reconnection are the CALLER's: ``subscribe`` raises on a broken +transport rather than retrying, because how long to wait before dialling again +is a policy that differs per broker (see the 429-storm note in ``ais_keyless``). +""" + +from __future__ import annotations + +import asyncio +import ssl +import time +from collections.abc import AsyncIterator +from urllib.parse import urlparse + +import websockets + +MQTT_KEEPALIVE_S = 60.0 +# Wake at least this often to send a PINGREQ, comfortably inside the keepalive. +_PING_EVERY_S = 25.0 + +# Packet types we act on. +_CONNACK = 2 +_PUBLISH = 3 +_SUBACK = 9 +_PINGREQ = b"\xc0\x00" + + +# ── wire codec ──────────────────────────────────────────────────────────────── + + +def enc_remaining_length(n: int) -> bytes: + out = bytearray() + while True: + b = n % 128 + n //= 128 + if n > 0: + b |= 0x80 + out.append(b) + if n == 0: + break + return bytes(out) + + +def connect_packet(client_id: str = "osint-geoint") -> bytes: + # variable header: proto name "MQTT", level 4, clean-session flag, keepalive 60 + vh = b"\x00\x04MQTT\x04\x02\x00\x3c" + payload = len(client_id).to_bytes(2, "big") + client_id.encode() + body = vh + payload + return b"\x10" + enc_remaining_length(len(body)) + body + + +def subscribe_packet(topic: str, packet_id: int = 1) -> bytes: + body = packet_id.to_bytes(2, "big") + len(topic).to_bytes(2, "big") + topic.encode() + b"\x00" + return b"\x82" + enc_remaining_length(len(body)) + body + + +def parse_packets(buf: bytes) -> tuple[list[tuple[int, int, bytes]], bytes]: + """Parse complete MQTT packets from ``buf``. + + Returns ``([(packet_type, byte0, body), …], remainder)``. A frame may carry + partial / multiple MQTT packets, so the caller accumulates the remainder + across reads. + """ + out: list[tuple[int, int, bytes]] = [] + i, n = 0, len(buf) + while i < n: + b0 = buf[i] + ptype = b0 >> 4 + mult, rl, j = 1, 0, i + 1 + while True: + if j >= n: + return out, buf[i:] # length incomplete + d = buf[j] + rl += (d & 0x7F) * mult + mult *= 128 + j += 1 + if not (d & 0x80): + break + if mult > 128**4: + return out, b"" # malformed; drop + if j + rl > n: + return out, buf[i:] # body incomplete + out.append((ptype, b0, buf[j : j + rl])) + i = j + rl + return out, b"" + + +def decode_publish(byte0: int, body: bytes) -> tuple[str, bytes] | None: + """Extract ``(topic, payload)`` from a PUBLISH packet body.""" + if len(body) < 2: + return None + qos = (byte0 >> 1) & 3 + tlen = int.from_bytes(body[0:2], "big") + if len(body) < 2 + tlen: + return None + topic = body[2 : 2 + tlen].decode("utf-8", "replace") + off = 2 + tlen + (2 if qos > 0 else 0) + return topic, body[off:] + + +# ── transports ──────────────────────────────────────────────────────────────── +# MQTT is the same byte stream either way; only how it is carried differs, so a +# two-method link is the whole abstraction. + + +class _Link: + async def send(self, data: bytes) -> None: # pragma: no cover - interface + raise NotImplementedError + + async def recv(self, wait_s: float) -> bytes | None: # pragma: no cover + raise NotImplementedError + + async def close(self) -> None: # pragma: no cover - interface + raise NotImplementedError + + +class _WsLink(_Link): + def __init__(self, ws: object) -> None: + self._ws = ws + + async def send(self, data: bytes) -> None: + await self._ws.send(data) # type: ignore[attr-defined] + + async def recv(self, wait_s: float) -> bytes | None: + try: + msg = await asyncio.wait_for(self._ws.recv(), timeout=wait_s) # type: ignore[attr-defined] + except TimeoutError: + return None + return msg if isinstance(msg, bytes) else str(msg).encode() + + async def close(self) -> None: + await self._ws.close() # type: ignore[attr-defined] + + +class _TcpLink(_Link): + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._r, self._w = reader, writer + + async def send(self, data: bytes) -> None: + self._w.write(data) + await self._w.drain() + + async def recv(self, wait_s: float) -> bytes | None: + try: + chunk = await asyncio.wait_for(self._r.read(65536), timeout=wait_s) + except TimeoutError: + return None + if chunk == b"": + raise ConnectionError("broker closed the connection") + return chunk + + async def close(self) -> None: + self._w.close() + try: + await self._w.wait_closed() + except OSError: + pass + + +async def _open(url: str) -> _Link: + parsed = urlparse(url) + scheme = (parsed.scheme or "").lower() + if scheme in ("ws", "wss"): + ctx = ssl.create_default_context() if scheme == "wss" else None + ws = await websockets.connect( + url, subprotocols=["mqtt"], ssl=ctx, ping_interval=None + ) + return _WsLink(ws) + if scheme in ("mqtt", "mqtts", "tcp"): + host = parsed.hostname + if not host: + raise ValueError(f"MQTT url has no host: {url!r}") + port = parsed.port or (8883 if scheme == "mqtts" else 1883) + ctx = ssl.create_default_context() if scheme == "mqtts" else None + reader, writer = await asyncio.open_connection(host, port, ssl=ctx) + return _TcpLink(reader, writer) + raise ValueError( + f"unsupported MQTT url scheme {scheme!r}: use mqtt, mqtts, ws or wss" + ) + + +async def subscribe( + url: str, topic: str, *, client_id: str = "osint-geoint" +) -> AsyncIterator[tuple[str, bytes]]: + """Yield ``(topic, payload)`` for every message on ``topic``. + + Runs until the caller stops consuming or the transport breaks, at which + point the underlying exception propagates — reconnect policy belongs to the + caller. + """ + link = await _open(url) + try: + await link.send(connect_packet(client_id)) + buf = b"" + subscribed = False + last_send = time.monotonic() + while True: + chunk = await link.recv(_PING_EVERY_S) + if chunk: + buf += chunk + packets, buf = parse_packets(buf) + for ptype, b0, body in packets: + if ptype == _CONNACK: + if len(body) > 1 and body[1] != 0: + raise ConnectionError( + f"broker refused the connection (code {body[1]})" + ) + if not subscribed: + await link.send(subscribe_packet(topic)) + last_send = time.monotonic() + elif ptype == _SUBACK: + subscribed = True + elif ptype == _PUBLISH: + pub = decode_publish(b0, body) + if pub is not None: + yield pub + if time.monotonic() - last_send > _PING_EVERY_S: + await link.send(_PINGREQ) + last_send = time.monotonic() + finally: + await link.close() diff --git a/apps/api/app/routes/actions.py b/apps/api/app/routes/actions.py index 7a00f5b6..4fa1b95e 100644 --- a/apps/api/app/routes/actions.py +++ b/apps/api/app/routes/actions.py @@ -14,12 +14,11 @@ from __future__ import annotations -import time -import uuid from typing import Any from fastapi import APIRouter, Depends, HTTPException +from app.intel import action_proposals_local from app.intel.actions import ActionResult, dispatch, list_actions from app.keys import UserCtx, current_user @@ -52,35 +51,25 @@ async def run_action( # When approval mode is ON (config.action_approval), the intel agent stores its # write-back actions here as PROPOSALS instead of dispatching them directly; the # operator approves/rejects in AgentConsole and approval executes through the -# SAME audited ``dispatch`` path above. In-memory + single-process: a restart -# drops pending proposals, which is acceptable — the agent re-proposes on its -# next run. -_PROPOSALS: dict[str, dict] = {} +# SAME audited ``dispatch`` path above. +# +# Persisted (``intel/action_proposals_local.py``), not a module dict. It WAS a +# dict, on the reasoning that "the agent re-proposes on its next run" — which +# only holds if the agent runs again. An operator who left proposals open +# overnight and restarted the backend came back to an empty queue with no +# record that anything had been waiting. PROPOSAL_TTL_S = 900 -def _prune_proposals() -> None: - cutoff = time.time() - PROPOSAL_TTL_S - for pid in [p for p, row in _PROPOSALS.items() if row["created"] < cutoff]: - _PROPOSALS.pop(pid, None) - - -def propose(name: str, params: dict, ctx, confidence: float = 0.0) -> str: +async def propose(name: str, params: dict, ctx, confidence: float = 0.0) -> str: """Queue action ``name`` with ``params`` for operator approval; returns its id.""" - _prune_proposals() - pid = uuid.uuid4().hex[:12] - _PROPOSALS[pid] = { - "id": pid, "name": name, "params": params, - "created": time.time(), "confidence": confidence, - } - return pid + return await action_proposals_local.add(name, params, confidence, PROPOSAL_TTL_S) @router.get("/api/actions/proposals") async def list_proposals(ctx: UserCtx = Depends(current_user)) -> list[dict]: """Pending proposals awaiting operator approval, oldest first.""" - _prune_proposals() - return sorted(_PROPOSALS.values(), key=lambda r: r["created"]) + return await action_proposals_local.list_pending(PROPOSAL_TTL_S) @router.post("/api/actions/proposals/{pid}/approve") @@ -90,9 +79,10 @@ async def approve_proposal(pid: str, ctx: UserCtx = Depends(current_user)): The audit row's actor is the approving ``ctx`` (``ctx.user_id``) — that is the fact that matters — so the approval is attributed without threading extras through dispatch. 404 for an unknown or expired proposal. + + ``take`` removes the row before dispatching, so a double-click approves once. """ - _prune_proposals() - row = _PROPOSALS.pop(pid, None) + row = await action_proposals_local.take(pid, PROPOSAL_TTL_S) if row is None: raise HTTPException(status_code=404, detail="unknown or expired proposal") return await dispatch(row["name"], row["params"], ctx) @@ -101,7 +91,6 @@ async def approve_proposal(pid: str, ctx: UserCtx = Depends(current_user)): @router.post("/api/actions/proposals/{pid}/reject") async def reject_proposal(pid: str, ctx: UserCtx = Depends(current_user)) -> dict: """Drop a queued proposal without executing it. 404 if unknown/expired.""" - row = _PROPOSALS.pop(pid, None) - if row is None: + if await action_proposals_local.take(pid, PROPOSAL_TTL_S) is None: raise HTTPException(status_code=404, detail="unknown or expired proposal") return {"ok": True, "id": pid} diff --git a/apps/api/app/routes/foundry.py b/apps/api/app/routes/foundry.py index 83367ada..cdf9a22d 100644 --- a/apps/api/app/routes/foundry.py +++ b/apps/api/app/routes/foundry.py @@ -13,7 +13,7 @@ import asyncio import json import time -from typing import Any +from typing import Any, Literal from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from pydantic import BaseModel, Field @@ -21,6 +21,7 @@ from app.config import get_settings from app.foundry import binding as binding_mod from app.foundry import builds as builds_mod +from app.foundry import connections as connections_mod from app.foundry import geo as geo_mod from app.foundry import ingest, sqlrun from app.foundry import seed as seed_mod @@ -355,6 +356,149 @@ async def upload_dataset_version( return ds +class ConnectionIn(BaseModel): + """A source the operator configured. ``config`` is per-kind: + + mqtt {url: "mqtt://host:1883" | "wss://host/mqtt", topic, client_id?} + kafka {bootstrap_servers, topic, group_id?, auto_offset_reset?} + sql {dsn_env, query, interval_s?} + + For ``sql``, ``dsn_env`` is the NAME of an environment variable holding the + connection string. Never the connection string: this row is returned by the + list route and lives in foundry.db, and a password belongs in neither. + """ + + name: str = Field(..., min_length=1, max_length=80) + kind: Literal["mqtt", "kafka", "sql"] + dataset_id: str = Field(..., min_length=1, max_length=80) + config: dict[str, Any] = Field(default_factory=dict) + enabled: bool = True + + +class ConnectionUpdate(BaseModel): + dataset_id: str = Field(..., min_length=1, max_length=80) + config: dict[str, Any] = Field(default_factory=dict) + enabled: bool = True + + +def _reject_inline_dsn(kind: str, config: dict[str, Any]) -> None: + """A SQL connection must not be handed a connection string. + + Caught at the boundary rather than at run time, because by the time the + runner reads it the value is already stored, already in the list response + and already in whatever backed the file up. + """ + if kind != "sql": + return + if not connections_mod.valid_dsn_env(str(config.get("dsn_env") or "")): + raise HTTPException( + status_code=422, + detail="dsn_env must be the NAME of an environment variable holding " + "the connection string (e.g. OSINT_SQL_DSN_WAREHOUSE), not the " + "connection string itself", + ) + + +@router.get("/api/foundry/connections") +async def list_connections( + ctx: UserCtx = Depends(current_user_or_local), +) -> dict[str, Any]: + """Configured sources, plus which kinds this deployment can run. + + ``availability`` is reported so the UI can grey out a kind rather than let + an operator configure one whose client is not installed. + """ + live = set(connections_mod.running_ids()) + rows = await _store().list_connections() + for row in rows: + row["running"] = row["id"] in live + return {"connections": rows, "availability": connections_mod.availability()} + + +@router.post("/api/foundry/connections") +async def create_connection( + body: ConnectionIn, ctx: UserCtx = Depends(current_user_or_local) +) -> dict[str, Any]: + _reject_inline_dsn(body.kind, body.config) + store = _store() + if await store.get_dataset(body.dataset_id) is None: + raise HTTPException(status_code=404, detail="dataset not found") + try: + created = await store.create_connection( + body.name, body.kind, body.dataset_id, body.config, body.enabled + ) + except FoundryError as exc: + _raise(exc) + raise AssertionError("unreachable") from exc # pragma: no cover + await connections_mod.reconcile() + return created + + +@router.put("/api/foundry/connections/{connection_id}") +async def update_connection( + connection_id: str, + body: ConnectionUpdate, + ctx: UserCtx = Depends(current_user_or_local), +) -> dict[str, Any]: + store = _store() + existing = await store.get_connection(connection_id) + if existing is None: + raise HTTPException(status_code=404, detail="connection not found") + _reject_inline_dsn(existing["kind"], body.config) + updated = await store.update_connection( + connection_id, + dataset_id=body.dataset_id, + config=body.config, + enabled=body.enabled, + ) + if updated is None: # pragma: no cover - raced with a delete + raise HTTPException(status_code=404, detail="connection not found") + await connections_mod.reconcile() + return updated + + +@router.delete("/api/foundry/connections/{connection_id}") +async def delete_connection( + connection_id: str, ctx: UserCtx = Depends(current_user_or_local) +) -> dict[str, Any]: + if not await _store().delete_connection(connection_id): + raise HTTPException(status_code=404, detail="connection not found") + await connections_mod.reconcile() + return {"deleted": connection_id} + + +@router.post("/api/foundry/datasets/{dataset_id}/ingest-token") +async def mint_ingest_token( + dataset_id: str, ctx: UserCtx = Depends(current_user_or_local) +) -> dict[str, Any]: + """Arm this dataset's inbound push endpoint and return the token. + + The token is shown HERE AND NOWHERE ELSE: only its sha256 is stored, so no + later response can hand it back. Calling this again rotates it and + invalidates the previous one. See ``routes/ingest.py`` for what it opens. + """ + token = await _store().mint_ingest_token(dataset_id) + if token is None: + raise HTTPException(status_code=404, detail="dataset not found") + return { + "dataset_id": dataset_id, + "token": token, + "url": f"/api/ingest/{dataset_id}", + "header": "X-Ingest-Token", + "note": "shown once; re-mint to rotate, DELETE to close the endpoint", + } + + +@router.delete("/api/foundry/datasets/{dataset_id}/ingest-token") +async def revoke_ingest_token( + dataset_id: str, ctx: UserCtx = Depends(current_user_or_local) +) -> dict[str, Any]: + """Close the dataset's push endpoint. Idempotent on an unarmed dataset.""" + if not await _store().clear_ingest_token(dataset_id): + raise HTTPException(status_code=404, detail="dataset not found") + return {"dataset_id": dataset_id, "ingest": "closed"} + + @router.post("/api/foundry/datasets/{dataset_id}/rollback") async def rollback_dataset( dataset_id: str, body: RollbackIn, ctx: UserCtx = Depends(current_user_or_local) diff --git a/apps/api/app/routes/ingest.py b/apps/api/app/routes/ingest.py new file mode 100644 index 00000000..8356f9fc --- /dev/null +++ b/apps/api/app/routes/ingest.py @@ -0,0 +1,128 @@ +"""Inbound push — ``POST /api/ingest/{dataset_id}`` (the one write from outside). + +Every other way data enters this platform is a pull the platform initiated: a +poller, a socket it dialled, a broker it subscribed to, an operator uploading a +file. Nothing could push. That is the whole Gotham "message queue / webhook" +connector category, and it is also the cheapest way to reach the ones this repo +will never carry a client for: a Kafka topic, an MQTT broker, a syslog tail, a +Zapier step and a shell script with `curl` all become the same ten lines on the +sender's side once a URL exists. + +It is deliberately not a new ingest path. A pushed body goes through the SAME +``append_version`` + ``auto_sync_dataset`` pair an upload does, so a row lands in +the ontology through whatever binding the operator already configured, obeys the +same row and byte caps, and shows up in the same version history. + +**This is the trust boundary.** The route does not use ``current_user_or_local`` +— a sender has no session — so a per-dataset bearer token is the entire gate: + + * the token is generated server-side and only its sha256 is stored, so a copy + of ``foundry.db`` is not a copy of the credential; + * it is compared with ``secrets.compare_digest``; + * it is never logged and never appears in any dataset response after the one + that mints it; + * a dataset without a token, and a dataset id that does not exist, answer with + the identical 404, so this cannot be used to discover dataset ids; + * the body is size-capped BEFORE it is parsed. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +from fastapi import APIRouter, Header, HTTPException, Request + +from app.config import get_settings +from app.foundry import binding as binding_mod +from app.foundry.store import MAX_UPLOAD_BYTES, FoundryError, FoundryStore +from app.keys import UserCtx + +router = APIRouter(tags=["ingest"]) +log = logging.getLogger("app.routes.ingest") + +# The token authenticates the SENDER, not a user, so the ontology write is +# attributed to the shared local identity — the same one a keyless boot uses. +_LOCAL_CTX = UserCtx(user_id="local", token="") + +# Same wording for "no such dataset" and "that dataset has no push endpoint". +_NO_ENDPOINT = "no ingest endpoint for this dataset" + + +async def _read_capped(request: Request) -> bytes: + """The body, refused at the cap rather than after it. + + A Content-Length check alone is not enough (it is absent under chunked + transfer encoding and it is the sender's claim either way), so the stream is + also totalled as it arrives and abandoned the moment it goes over. + """ + declared = request.headers.get("content-length") + if declared and declared.isdigit() and int(declared) > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=413, detail=f"body too large: cap is {MAX_UPLOAD_BYTES} bytes" + ) + chunks: list[bytes] = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > MAX_UPLOAD_BYTES: + raise HTTPException( + status_code=413, + detail=f"body too large: cap is {MAX_UPLOAD_BYTES} bytes", + ) + chunks.append(chunk) + return b"".join(chunks) + + +@router.post("/api/ingest/{dataset_id}") +async def push_rows( + dataset_id: str, + request: Request, + x_ingest_token: str | None = Header(None), +) -> dict[str, Any]: + """Append one object, or an array of objects, to a dataset. + + Arm the endpoint first with ``POST /api/foundry/datasets/{id}/ingest-token`` + and send the token it returns as ``X-Ingest-Token``. + """ + settings = get_settings() + store = FoundryStore(settings) + + verdict = await store.ingest_token_matches(dataset_id, x_ingest_token or "") + if verdict is None: + raise HTTPException(status_code=404, detail=_NO_ENDPOINT) + if not verdict: + # No dataset id, no token prefix, nothing an attacker can grep a log for. + log.warning("rejected an ingest push with a bad token") + raise HTTPException(status_code=401, detail="invalid ingest token") + + raw = await _read_capped(request) + try: + body = json.loads(raw or b"null") + except (json.JSONDecodeError, ValueError) as exc: + raise HTTPException(status_code=422, detail=f"body is not JSON: {exc}") from exc + + if isinstance(body, dict): + rows = [body] + elif isinstance(body, list): + rows = body + else: + raise HTTPException( + status_code=422, detail="body must be an object or an array of objects" + ) + if any(not isinstance(r, dict) for r in rows): + raise HTTPException( + status_code=422, detail="body must be an object or an array of objects" + ) + + try: + result = await store.append_version(dataset_id, rows) + except FoundryError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + # The token authenticates the SENDER, not a user, so the ontology write is + # attributed to the shared local identity — the same one a keyless boot uses. + result["auto_sync"] = await binding_mod.auto_sync_dataset(store, dataset_id, _LOCAL_CTX) + result["rows_added"] = len(rows) + return result diff --git a/apps/api/app/routes/ontology.py b/apps/api/app/routes/ontology.py index 11fdca79..5a7fb6a1 100644 --- a/apps/api/app/routes/ontology.py +++ b/apps/api/app/routes/ontology.py @@ -7,6 +7,8 @@ named investigation as an ontology node — a graph-shaping write, not a kinetic action, so it needs no ``action_log`` audit row. + GET /api/ontology/schema → declared relations + kind props + GET /api/ontology/search?q=&kind= → full-text over the stored graph GET /api/ontology/object/{id} → one Object (404 if absent) POST /api/ontology/object → upsert one Object (save a node) GET /api/ontology/assertions/{id}?prop= → the id's assertion history @@ -35,10 +37,25 @@ SearchAround, get_registry, ) +from app.intel.ontology_schema import schema_payload, validate_object from app.keys import UserCtx, current_user_or_local router = APIRouter(tags=["ontology"]) + +class ObjectSaved(Object): + """What ``POST /api/ontology/object`` answers: the stored object, plus any + way it departs from what its kind declares. + + A subclass rather than a wrapper so the round-trip contract the Investigation + canvas depends on is untouched — every ``Object`` field is still at the top + level, and a caller that does not know about ``warnings`` reads the same + body it always did. The warnings are advisory and are NOT stored: the + registry accepts the write either way (see ``intel/ontology_schema.py``). + """ + + warnings: list[str] = Field(default_factory=list) + # Trigger → provenance source for POST /api/ontology/promote. The SERVER owns # the source string (the client passes only a trigger enum, never a raw source) # so a caller can't forge feed/rule authority into the assertion trail — @@ -90,10 +107,41 @@ async def object_assertions( return await reg.get_assertions(object_id, prop=prop, limit=limit) -@router.post("/api/ontology/object", response_model=Object) +@router.get("/api/ontology/schema") +async def ontology_schema() -> dict[str, Any]: + """What the ontology declares: every relation with BOTH of its names, and + the properties each known kind carries. + + Static, per-deployment, and identical for every caller, so it needs no auth + dependency and no store round-trip. The Graph canvas reads it once to label + an edge correctly when it is traversed from the target's end; the Explorer + reads it to build typed facets. + """ + return schema_payload() + + +@router.get("/api/ontology/search", response_model=list[Object]) +async def search_objects( + q: str = Query(..., min_length=1, max_length=200), + kind: list[str] | None = Query(None), + limit: int = Query(50, ge=1, le=200), + ctx: UserCtx = Depends(current_user_or_local), +) -> list[Object]: + """Find ontology objects by words in their id, kind or property values. + + Distinct from ``/api/search/objects``, which searches the LIVE observation + store — what is being emitted right now. This searches what was promoted + into the graph and kept, which until now had no search at all: ``get`` needs + the exact canonical id and ``list_by_kind`` filters one props field. + """ + reg = get_registry(ctx, get_settings()) + return await reg.search(q, kinds=kind, limit=limit) + + +@router.post("/api/ontology/object", response_model=ObjectSaved) async def upsert_object( obj: Object, ctx: UserCtx = Depends(current_user_or_local) -) -> Object: +) -> ObjectSaved: """Insert or merge one ontology object (RLS-scoped to the caller). The graph-shaping write the Investigation canvas (C4) uses to persist a saved @@ -102,9 +150,16 @@ async def upsert_object( (``upsert`` calls ``normalised()``), so a caller may omit it. This is NOT a kinetic action — no ``action_log`` audit row — so it stays here rather than in ``/api/actions``. + + The write happens first and unconditionally: ``warnings`` describes the + object that was stored, it does not gate storing it. """ reg = get_registry(ctx, get_settings()) - return await reg.upsert(obj) + saved = await reg.upsert(obj) + return ObjectSaved( + **saved.model_dump(), + warnings=validate_object(saved.kind, saved.props), + ) class PromoteIn(BaseModel): diff --git a/apps/api/app/routes/situations.py b/apps/api/app/routes/situations.py index da9893f8..c20e8ebd 100644 --- a/apps/api/app/routes/situations.py +++ b/apps/api/app/routes/situations.py @@ -36,7 +36,8 @@ from app import llm from app.config import get_settings from app.intel import case_export -from app.intel.ontology import Link, Object, get_registry +from app.intel.ontology import Link, Object, get_registry, kind_of +from app.intel.ontology_schema import validate_link from app.keys import UserCtx, current_user_or_local router = APIRouter(tags=["situations"]) @@ -233,10 +234,21 @@ async def delete_situation(sit_id: str, ctx: UserCtx = Depends(current_user_or_l await reg.delete(sit_id) -@router.post("/api/situations/{sit_id:path}/link", response_model=Link) +class LinkSaved(Link): + """The stored link plus any way it departs from what its relation declares. + + Same shape rule as ``routes/ontology.py::ObjectSaved``: a subclass, so every + ``Link`` field stays at the top level and an older caller reads the body it + always did. Advisory only, never stored, never a reason to refuse the write. + """ + + warnings: list[str] = Field(default_factory=list) + + +@router.post("/api/situations/{sit_id:path}/link", response_model=LinkSaved) async def link_child( sit_id: str, body: LinkIn, ctx: UserCtx = Depends(current_user_or_local) -) -> Link: +) -> LinkSaved: """Attach a child to a situation: ``situation --rel--> dst``. The one relationship-write the Situation/COA feature needs (``routes/ontology.py`` @@ -250,7 +262,10 @@ async def link_child( # can't see it. assert_props with empty props still mints the row (existence), # while the link above carries the provenance of *why* it was pulled in. await reg.assert_props(body.dst, {}, source="analyst:situation") - return link + return LinkSaved( + **link.model_dump(), + warnings=validate_link(link.rel, kind_of(link.src), kind_of(link.dst)), + ) # ── case → report export (P2) ───────────────────────────────────────────────── diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 4ad2fd76..cc06c9af 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -63,6 +63,18 @@ dev = [ "pytest-xdist>=3.8.0", "httpx[socks]>=0.27.2", "ruff>=0.6.8", + # Not a runtime dependency — see the `sql` extra below, which is what a + # deployment installs. It is here so the SQL connection can be PROVEN in CI + # rather than only configured: SQLAlchemy drives SQLite, so the whole path + # (env var → engine → query → dataset → ontology) runs with no server. + # Without this line tests/test_connections_sql.py skips and the documented + # backend baseline shifts by four between machines. + "sqlalchemy>=2.0", + # Same reason, smaller claim: aiokafka proves the import guard's POSITIVE + # branch and that an unreachable broker is recorded rather than fatal. The + # Kafka WIRE path stays unproven — see apps/api/CLAUDE.md — because faking a + # consumer-group handshake would test our misunderstanding of it. + "aiokafka>=0.11", ] # MAVLink bridge sidecar (app.mavlink_bridge). Optional: without it the bridge # runs log-only (echoes planned commands, no vehicle uplink). Install with @@ -70,6 +82,21 @@ dev = [ mavlink = [ "pymavlink>=2.4.40", ] +# Foundry connections (app.foundry.connections). Each kind is optional and +# reports itself unavailable when its client is absent, so a keyless boot with +# neither installed still works — that is a product requirement, not a dev +# convenience, and a deployment that does not consume Kafka must not be made to +# install a Kafka client. MQTT needs nothing: app/mqtt_client.py speaks the +# wire protocol over the websockets dependency that is already here. +# pip install -e '.[kafka]' Kafka topic → dataset +# pip install -e '.[sql]' SQL query → dataset (plus a DBAPI driver for +# the database in question, e.g. psycopg[binary]) +kafka = [ + "aiokafka>=0.11", +] +sql = [ + "sqlalchemy>=2.0", +] [build-system] requires = ["hatchling"] diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py index 73d9c3b8..49607c60 100644 --- a/apps/api/tests/conftest.py +++ b/apps/api/tests/conftest.py @@ -193,6 +193,20 @@ def _isolate_action_log_db(tmp_path: Path) -> Iterator[None]: action_log_local.override_db_path(None) +@pytest.fixture(autouse=True) +def _isolate_action_proposals_db(tmp_path: Path) -> Iterator[None]: + """Per-test temp file for the HITL proposal queue (mirrors action_log). + + It became a real file when the queue stopped being a module dict; without + this, a test that queues a proposal would leave it in the repo's data dir + and the next test would see it as pending.""" + from app.intel import action_proposals_local + + action_proposals_local.override_db_path(str(tmp_path / "action_proposals.db")) + yield + action_proposals_local.override_db_path(None) + + @pytest.fixture(autouse=True) def _isolate_evidence_dir(tmp_path: Path) -> Iterator[None]: """Point the evidence-locker blob dir at a per-test temp dir (mirrors the diff --git a/apps/api/tests/test_action_proposals.py b/apps/api/tests/test_action_proposals.py index ec99133a..8741ea0d 100644 --- a/apps/api/tests/test_action_proposals.py +++ b/apps/api/tests/test_action_proposals.py @@ -5,9 +5,16 @@ agent stores a PROPOSAL that the operator approves/rejects in AgentConsole. Approval executes through the SAME audited ``intel/actions.dispatch`` path. +The queue is PERSISTED (``intel/action_proposals_local.py``). It was a module +dict, which meant a restart silently emptied the approval queue — an operator +who left proposals open overnight came back to none, with no record that +anything had been waiting. ``test_a_restart_does_not_empty_the_queue`` is the +guard for that; the rest is behaviour that must survive the move. + These tests are fully hermetic — ``dispatch`` is monkeypatched so no Supabase / -ontology is touched, and the routes are exercised in-process with ``ctx=None`` -(the keyless path, exactly as the other route tests in this suite do). +ontology is touched, the queue DB is a per-test temp file (conftest), and the +routes are exercised in-process with ``ctx=None`` (the keyless path, exactly as +the other route tests in this suite do). """ from __future__ import annotations @@ -16,55 +23,122 @@ import pytest +from app.intel import action_proposals_local as store from app.routes import actions as actions_mod +# Captured before any test patches time.time, so "later than the TTL" is a +# fixed point rather than something that drifts with the clock under the patch. +_T0 = time.time() + + +def _later(ttl: float) -> float: + return _T0 + ttl + 60 + + +@pytest.mark.anyio +async def test_propose_stores_and_lists() -> None: + pid = await actions_mod.propose( + "flag_entity", {"entity_id": "vessel:1"}, ctx=None, confidence=0.4 + ) + rows = await actions_mod.list_proposals(ctx=None) + assert [r["id"] for r in rows] == [pid] + assert rows[0]["name"] == "flag_entity" + assert rows[0]["params"] == {"entity_id": "vessel:1"} + assert rows[0]["confidence"] == 0.4 + + +@pytest.mark.anyio +async def test_proposals_list_oldest_first() -> None: + first = await actions_mod.propose("flag_entity", {"n": 1}, ctx=None) + second = await actions_mod.propose("flag_entity", {"n": 2}, ctx=None) + rows = await actions_mod.list_proposals(ctx=None) + assert [r["id"] for r in rows] == [first, second] + + +@pytest.mark.anyio +async def test_a_restart_does_not_empty_the_queue() -> None: + """The reason this store exists. Nothing is held in the process, so a fresh + read against the same file still sees the pending work.""" + pid = await actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None) + # A restart is exactly this: no in-process state, same file on disk. + rows = await store.list_pending(actions_mod.PROPOSAL_TTL_S) + assert [r["id"] for r in rows] == [pid] -@pytest.fixture(autouse=True) -def _clean_proposals(): - actions_mod._PROPOSALS.clear() - yield - actions_mod._PROPOSALS.clear() +@pytest.mark.anyio +async def test_expired_proposal_is_not_listed(monkeypatch) -> None: # type: ignore[no-untyped-def] + pid = await actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None) + monkeypatch.setattr(time, "time", lambda: _later(actions_mod.PROPOSAL_TTL_S)) + assert await actions_mod.list_proposals(ctx=None) == [] + assert await store.take(pid, actions_mod.PROPOSAL_TTL_S) is None -def test_propose_stores_and_lists(): - pid = actions_mod.propose("flag_entity", {"entity_id": "vessel:1"}, ctx=None, confidence=0.4) - assert pid in actions_mod._PROPOSALS - row = actions_mod._PROPOSALS[pid] - assert row["name"] == "flag_entity" - assert row["params"] == {"entity_id": "vessel:1"} - assert row["confidence"] == 0.4 +@pytest.mark.anyio +async def test_expiry_is_enforced_on_read_not_only_on_prune(monkeypatch) -> None: # type: ignore[no-untyped-def] + """After a restart nothing has pruned yet, so a row that aged out while the + process was down must still not come back looking live.""" + await actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None) + monkeypatch.setattr(time, "time", lambda: _later(actions_mod.PROPOSAL_TTL_S)) + assert await store.list_pending(actions_mod.PROPOSAL_TTL_S) == [] -def test_expired_proposal_pruned(): - pid = actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None, confidence=0.0) - actions_mod._PROPOSALS[pid]["created"] = time.time() - actions_mod.PROPOSAL_TTL_S - 1 - actions_mod._prune_proposals() - assert pid not in actions_mod._PROPOSALS +@pytest.mark.anyio +async def test_prune_removes_expired_rows(monkeypatch) -> None: # type: ignore[no-untyped-def] + await actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None) + monkeypatch.setattr(time, "time", lambda: _later(actions_mod.PROPOSAL_TTL_S)) + assert await store.prune(actions_mod.PROPOSAL_TTL_S) == 1 + assert await store.prune(actions_mod.PROPOSAL_TTL_S) == 0 -@pytest.mark.asyncio -async def test_approve_executes_and_removes(monkeypatch): + +@pytest.mark.anyio +async def test_approve_executes_and_removes(monkeypatch) -> None: # type: ignore[no-untyped-def] calls: list[tuple] = [] - async def fake_dispatch(name, params, ctx): + async def fake_dispatch(name, params, ctx): # type: ignore[no-untyped-def] calls.append((name, params)) return {"ok": True, "action": name} monkeypatch.setattr(actions_mod, "dispatch", fake_dispatch) - pid = actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None, confidence=0.0) + pid = await actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None) result = await actions_mod.approve_proposal(pid, ctx=None) assert calls == [("flag_entity", {"entity_id": "v"})] - assert pid not in actions_mod._PROPOSALS + assert await actions_mod.list_proposals(ctx=None) == [] assert result["ok"] is True -@pytest.mark.asyncio -async def test_reject_removes_without_execute(monkeypatch): - async def boom(name, params, ctx): # must never run +@pytest.mark.anyio +async def test_approving_twice_executes_once(monkeypatch) -> None: # type: ignore[no-untyped-def] + """The row is taken before dispatch, so a double-click cannot run the same + write-back twice.""" + calls: list[str] = [] + + async def fake_dispatch(name, params, ctx): # type: ignore[no-untyped-def] + calls.append(name) + return {"ok": True} + + monkeypatch.setattr(actions_mod, "dispatch", fake_dispatch) + pid = await actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None) + await actions_mod.approve_proposal(pid, ctx=None) + with pytest.raises(Exception) as exc: + await actions_mod.approve_proposal(pid, ctx=None) + assert getattr(exc.value, "status_code", None) == 404 + assert calls == ["flag_entity"] + + +@pytest.mark.anyio +async def test_reject_removes_without_execute(monkeypatch) -> None: # type: ignore[no-untyped-def] + async def boom(name, params, ctx): # type: ignore[no-untyped-def] raise AssertionError("dispatch called on reject") monkeypatch.setattr(actions_mod, "dispatch", boom) - pid = actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None, confidence=0.0) + pid = await actions_mod.propose("flag_entity", {"entity_id": "v"}, ctx=None) out = await actions_mod.reject_proposal(pid, ctx=None) assert out == {"ok": True, "id": pid} - assert pid not in actions_mod._PROPOSALS + assert await actions_mod.list_proposals(ctx=None) == [] + + +@pytest.mark.anyio +async def test_unknown_id_is_a_404() -> None: + with pytest.raises(Exception) as exc: + await actions_mod.reject_proposal("nope", ctx=None) + assert getattr(exc.value, "status_code", None) == 404 diff --git a/apps/api/tests/test_connections.py b/apps/api/tests/test_connections.py new file mode 100644 index 00000000..8cf8e9b4 --- /dev/null +++ b/apps/api/tests/test_connections.py @@ -0,0 +1,312 @@ +"""Guards for operator-configured sources (foundry/connections.py). + +Three things are worth pinning and one of them is the reason the file exists: + +1. **A keyless boot survives a missing optional client.** Absence is SIMULATED + with a sys.modules sentinel rather than trusted to the dev venv, because the + moment ``aiokafka`` is installed here a test that merely imports it would + pass while proving nothing. +2. **A SQL connection can never hold a connection string.** The row is returned + by the list route and sits in foundry.db, so a password in it is a leak with + several copies. +3. The supervisor reconciles, rather than starting things once. +""" + +from __future__ import annotations + +import asyncio +import builtins +import json + +import pytest +from fastapi.testclient import TestClient + +from app.foundry import connections as C + + +@pytest.fixture +def dataset(client: TestClient) -> str: + r = client.post( + "/api/foundry/datasets/upload", + files={"file": ("seed.csv", b"a\n1\n", "text/csv")}, + data={"name": "conn_target"}, + ) + assert r.status_code == 200, r.text + return r.json()["id"] + + +@pytest.fixture(autouse=True) +def _no_stray_tasks(): # type: ignore[no-untyped-def] + yield + C._tasks.clear() + C._fingerprints.clear() + + +# ── optional dependencies ───────────────────────────────────────────────────── + + +def _hide(monkeypatch, module: str) -> None: # type: ignore[no-untyped-def] + """Make ``import `` fail even when it is installed.""" + real_import = builtins.__import__ + + def _fake(name, *args, **kwargs): # type: ignore[no-untyped-def] + if name == module or name.startswith(module + "."): + raise ModuleNotFoundError(f"No module named {module!r}") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _fake) + + +def test_mqtt_never_needs_an_install() -> None: + assert C.availability()["mqtt"]["available"] is True + + +def test_probe_says_available_for_something_importable() -> None: + """The positive half. Without it, a ``_probe`` that always reported absence + would pass every other test in this section.""" + assert C._probe("json") is None + assert C._probe("a_module_that_does_not_exist") is not None + + +@pytest.mark.anyio +async def test_the_mqtt_runner_refuses_an_incomplete_config() -> None: + with pytest.raises(ValueError, match="url and a topic"): + await C._run_mqtt(None, {"config": {"url": "mqtt://h:1883"}}) # type: ignore[arg-type] + + +@pytest.mark.anyio +async def test_the_sql_runner_refuses_an_empty_query(monkeypatch) -> None: # type: ignore[no-untyped-def] + import sys + import types + + # sqlalchemy is not installed here, so stand one in: the assertion is about + # the runner's own validation, not about the driver. + monkeypatch.setitem(sys.modules, "sqlalchemy", types.ModuleType("sqlalchemy")) + with pytest.raises(ValueError, match="needs a query"): + await C._run_sql(None, {"config": {"dsn_env": "X", "query": " "}}) # type: ignore[arg-type] + + +def test_kafka_reports_unavailable_when_its_client_is_absent(monkeypatch) -> None: # type: ignore[no-untyped-def] + _hide(monkeypatch, "aiokafka") + kafka = C.availability()["kafka"] + assert kafka["available"] is False + assert "aiokafka" in kafka["detail"] + + +def test_sql_reports_unavailable_when_its_client_is_absent(monkeypatch) -> None: # type: ignore[no-untyped-def] + _hide(monkeypatch, "sqlalchemy") + sql = C.availability()["sql"] + assert sql["available"] is False + assert "sqlalchemy" in sql["detail"] + + +def test_the_route_reports_availability(client: TestClient) -> None: + body = client.get("/api/foundry/connections").json() + assert set(body["availability"]) == {"mqtt", "kafka", "sql"} + + +@pytest.mark.anyio +async def test_reconcile_skips_a_kind_whose_client_is_missing( + client: TestClient, dataset: str, monkeypatch +) -> None: # type: ignore[no-untyped-def] + """The keyless-boot guard: an unavailable kind is left unstarted instead of + crashing the supervisor.""" + client.post( + "/api/foundry/connections", + json={ + "name": "k", + "kind": "kafka", + "dataset_id": dataset, + "config": {"bootstrap_servers": "localhost:9092", "topic": "t"}, + }, + ) + _hide(monkeypatch, "aiokafka") + C._tasks.clear() + await C.reconcile() + assert C.running_ids() == [] + + +# ── the DSN never lands in the database ─────────────────────────────────────── + + +@pytest.mark.parametrize( + "dsn_env", + [ + "postgresql://user:hunter2@db.internal/prod", + "postgres://localhost/x", + "lower_case_name", + "HAS SPACE", + "", + ], +) +def test_a_sql_connection_refuses_anything_but_a_variable_name( + client: TestClient, dataset: str, dsn_env: str +) -> None: + r = client.post( + "/api/foundry/connections", + json={ + "name": f"sql-{abs(hash(dsn_env))}", + "kind": "sql", + "dataset_id": dataset, + "config": {"dsn_env": dsn_env, "query": "SELECT 1"}, + }, + ) + assert r.status_code == 422, r.text + assert "environment variable" in r.json()["detail"] + + +def test_a_variable_name_is_accepted(client: TestClient, dataset: str) -> None: + r = client.post( + "/api/foundry/connections", + json={ + "name": "warehouse", + "kind": "sql", + "dataset_id": dataset, + "config": {"dsn_env": "OSINT_SQL_DSN_WAREHOUSE", "query": "SELECT 1"}, + }, + ) + assert r.status_code == 200, r.text + assert r.json()["config"]["dsn_env"] == "OSINT_SQL_DSN_WAREHOUSE" + + +def test_no_listing_can_contain_a_password(client: TestClient, dataset: str) -> None: + client.post( + "/api/foundry/connections", + json={ + "name": "warehouse", + "kind": "sql", + "dataset_id": dataset, + "config": {"dsn_env": "OSINT_SQL_DSN_WAREHOUSE", "query": "SELECT 1"}, + }, + ) + listed = json.dumps(client.get("/api/foundry/connections").json()) + assert "hunter2" not in listed + assert "://" not in listed + + +def test_an_unset_variable_is_a_readable_error_not_a_crash(monkeypatch) -> None: # type: ignore[no-untyped-def] + monkeypatch.delenv("OSINT_SQL_DSN_NOPE", raising=False) + with pytest.raises(ValueError, match="OSINT_SQL_DSN_NOPE is not set"): + C._resolve_dsn({"dsn_env": "OSINT_SQL_DSN_NOPE"}) + + +def test_a_driver_error_carrying_the_dsn_is_scrubbed() -> None: + """SQLAlchemy puts the URL in some exception messages. Whatever reaches + last_error must not.""" + secret = "postgresql://user:hunter2@db/prod" + assert C._scrub(f"OperationalError: could not connect to {secret}", secret) == ( + "OperationalError: could not connect to ***" + ) + + +# ── CRUD and supervision ────────────────────────────────────────────────────── + + +def test_create_requires_a_real_dataset(client: TestClient) -> None: + r = client.post( + "/api/foundry/connections", + json={ + "name": "x", + "kind": "mqtt", + "dataset_id": "ds_nope", + "config": {"url": "mqtt://h:1883", "topic": "t"}, + }, + ) + assert r.status_code == 404 + + +def test_duplicate_names_are_refused(client: TestClient, dataset: str) -> None: + body = { + "name": "dupe", + "kind": "mqtt", + "dataset_id": dataset, + "config": {"url": "mqtt://h:1883", "topic": "t"}, + "enabled": False, + } + assert client.post("/api/foundry/connections", json=body).status_code == 200 + assert client.post("/api/foundry/connections", json=body).status_code == 409 + + +def test_update_and_delete(client: TestClient, dataset: str) -> None: + created = client.post( + "/api/foundry/connections", + json={ + "name": "edit-me", + "kind": "mqtt", + "dataset_id": dataset, + "config": {"url": "mqtt://h:1883", "topic": "a"}, + "enabled": False, + }, + ).json() + updated = client.put( + f"/api/foundry/connections/{created['id']}", + json={"dataset_id": dataset, "config": {"url": "mqtt://h:1883", "topic": "b"}, "enabled": False}, + ).json() + assert updated["config"]["topic"] == "b" + assert client.delete(f"/api/foundry/connections/{created['id']}").status_code == 200 + assert client.delete(f"/api/foundry/connections/{created['id']}").status_code == 404 + + +@pytest.mark.anyio +async def test_reconcile_starts_stops_and_restarts_on_an_edit( + client: TestClient, dataset: str, monkeypatch +) -> None: # type: ignore[no-untyped-def] + """A connection that dies at 03:00 must not stay dead until a reboot, and an + edit in the UI must take effect without one.""" + seen: list[str] = [] + + async def _fake_runner(store, conn): # type: ignore[no-untyped-def] + seen.append(conn["config"]["topic"]) + await asyncio.Event().wait() + + monkeypatch.setitem(C._RUNNERS, "mqtt", _fake_runner) + created = client.post( + "/api/foundry/connections", + json={ + "name": "live", + "kind": "mqtt", + "dataset_id": dataset, + "config": {"url": "mqtt://h:1883", "topic": "first"}, + }, + ).json() + + await C.reconcile() + await asyncio.sleep(0) + assert C.running_ids() == [created["id"]] + + client.put( + f"/api/foundry/connections/{created['id']}", + json={"dataset_id": dataset, "config": {"url": "mqtt://h:1883", "topic": "second"}, "enabled": True}, + ) + await C.reconcile() + await asyncio.sleep(0) + assert seen == ["first", "second"] + + client.put( + f"/api/foundry/connections/{created['id']}", + json={"dataset_id": dataset, "config": {"url": "mqtt://h:1883", "topic": "second"}, "enabled": False}, + ) + await C.reconcile() + assert C.running_ids() == [] + + +# ── message shaping ─────────────────────────────────────────────────────────── + + +def test_a_json_object_message_becomes_the_row() -> None: + row = C.message_row("vessels/1", b'{"mmsi": 1, "sog": 12.5}') + assert row["mmsi"] == 1 + assert row["sog"] == 12.5 + assert row["_topic"] == "vessels/1" + + +def test_a_message_that_is_not_a_json_object_is_kept_verbatim() -> None: + """Dropping it would hide exactly the message an operator needs to see to + fix their topic.""" + assert C.message_row("t", b"not json")["payload"] == "not json" + assert C.message_row("t", b"[1,2]")["payload"] == "[1,2]" + + +def test_a_message_carrying_its_own_topic_field_keeps_it() -> None: + row = C.message_row("broker/topic", b'{"_topic": "mine"}') + assert row["_topic"] == "mine" diff --git a/apps/api/tests/test_connections_kafka.py b/apps/api/tests/test_connections_kafka.py new file mode 100644 index 00000000..c865024c --- /dev/null +++ b/apps/api/tests/test_connections_kafka.py @@ -0,0 +1,91 @@ +"""What can be proven about the Kafka connection without a broker. + +Not the wire. Kafka's protocol is a consumer-group handshake across half a dozen +request types, and a fake broker good enough to satisfy aiokafka would be more +likely to encode our own misunderstanding than to catch one — so +``apps/api/CLAUDE.md`` records the wire path as unproven and says why. MQTT got +an in-test broker because its protocol is four packet types; Kafka does not. + +Two things do not need a broker and were assumed until now: + + * the import guard's POSITIVE branch — that ``availability()`` says available + when the client is installed. Every other availability test asserts the + absent case, so a probe that always answered "unavailable" would have + passed all of them; + * that a broker which cannot be reached is RECORDED on the connection and + retried, rather than taking the supervisor down with it. That is the only + Kafka behaviour an operator will meet if they mistype a hostname, and it is + reachable by pointing the runner at a closed port. + +Skipped when the optional extra is absent, which is the expected keyless state. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from app.foundry import connections as C + +pytest.importorskip("aiokafka", reason="optional extra: pip install -e '.[kafka]'") + + +def test_availability_reports_kafka_as_present_when_it_is_installed() -> None: + assert C.availability()["kafka"] == {"available": True, "detail": "aiokafka"} + + +@pytest.mark.anyio +async def test_an_incomplete_config_is_refused_before_dialling() -> None: + with pytest.raises(ValueError, match="bootstrap_servers and a topic"): + await C._run_kafka(None, {"config": {"topic": "t"}}) # type: ignore[arg-type] + with pytest.raises(ValueError, match="bootstrap_servers and a topic"): + await C._run_kafka(None, {"config": {"bootstrap_servers": "h:9092"}}) # type: ignore[arg-type] + + +@pytest.mark.anyio +async def test_an_unreachable_broker_is_recorded_and_retried(client) -> None: # type: ignore[no-untyped-def] + """A mistyped hostname is the failure an operator will actually hit. It has + to land on the row, not in a traceback that kills the supervisor.""" + ds = client.post( + "/api/foundry/datasets/upload", + files={"file": ("seed.csv", b"a\n1\n", "text/csv")}, + data={"name": "kafka_target"}, + ).json()["id"] + created = client.post( + "/api/foundry/connections", + json={ + "name": "unreachable", + "kind": "kafka", + "dataset_id": ds, + # Port 1 is closed; no broker, no DNS lookup, no waiting on egress. + "config": {"bootstrap_servers": "127.0.0.1:1", "topic": "t"}, + "enabled": False, + }, + ).json() + + from app.config import get_settings + from app.foundry.store import FoundryStore + + store = FoundryStore(get_settings()) + # Shorten the backoff so the retry is observable inside a test. + original = C._BACKOFF_START_S + C._BACKOFF_START_S = 0.05 + task = asyncio.create_task(C._run_forever({**created})) + try: + for _ in range(200): + await asyncio.sleep(0.05) + row = await store.get_connection(created["id"]) + if row and row["last_error"]: + break + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + C._BACKOFF_START_S = original + + row = await store.get_connection(created["id"]) + assert row is not None + assert row["last_error"], "an unreachable broker left no trace on the connection" + # The supervisor's task survived long enough to record and loop, which is + # the behaviour under test: a raised exception would have ended it instead. + assert row["last_ok"] is None diff --git a/apps/api/tests/test_connections_sql.py b/apps/api/tests/test_connections_sql.py new file mode 100644 index 00000000..fde3d3bb --- /dev/null +++ b/apps/api/tests/test_connections_sql.py @@ -0,0 +1,163 @@ +"""The SQL connection, against a real database engine. + +``test_connections.py`` proves the configuration boundary — that a connection +string can never be stored in place of an environment-variable name — but it +never opens a database. This does, using SQLAlchemy against SQLite, which is a +real engine driven by the real code path and needs no server to exist. + +What that buys over a mock: the DSN actually resolves from the environment, the +query actually executes through ``exec_driver_sql``, ``.mappings()`` actually +produces the row dicts the batcher expects, and a driver error actually carries +whatever SQLAlchemy chooses to put in it — which is the thing the scrubber has +to defeat. + +Skipped rather than failed when the optional extra is absent: ``sqlalchemy`` is +an opt-in install and a keyless deployment is expected not to have it. +""" + +from __future__ import annotations + +import asyncio +import sqlite3 + +import pytest + +from app.foundry import connections as C + +sqlalchemy = pytest.importorskip("sqlalchemy", reason="optional extra: pip install -e '.[sql]'") + + +@pytest.fixture +def source_db(tmp_path): # type: ignore[no-untyped-def] + """A database standing in for the operator's own, with rows to pull.""" + path = tmp_path / "warehouse.db" + con = sqlite3.connect(path) + con.execute("CREATE TABLE sites (mmsi INTEGER, name TEXT, lat REAL, lon REAL)") + con.executemany( + "INSERT INTO sites VALUES (?,?,?,?)", + [(636092333, "SQL ROW ONE", 51.9, 4.4), (636092444, "SQL ROW TWO", 30.5, 32.3)], + ) + con.commit() + con.close() + return f"sqlite:///{path}" + + +def test_availability_reports_sql_as_present_when_it_is_installed() -> None: + """The positive half of the optional-dependency contract. Without this, a + probe that always answered 'unavailable' would satisfy every other test.""" + assert C.availability()["sql"] == {"available": True, "detail": "sqlalchemy"} + + +@pytest.mark.anyio +async def test_a_query_lands_rows_and_mints_ontology_objects( + source_db: str, client, monkeypatch +) -> None: # type: ignore[no-untyped-def] + """The whole path: env var → engine → query → dataset version → binding → + ontology object.""" + monkeypatch.setenv("OSINT_SQL_DSN_TEST", source_db) + ds = client.post( + "/api/foundry/datasets/upload", + files={"file": ("seed.csv", b"mmsi,name\n1,SEED\n", "text/csv")}, + data={"name": "sql_target"}, + ).json()["id"] + assert client.post( + "/api/foundry/bindings", + json={ + "dataset_id": ds, + "object_kind": "vessel", + "key_column": "mmsi", + "prop_map": {"name": "name"}, + }, + ).status_code == 200 + + from app.config import get_settings + from app.foundry.store import FoundryStore + + conn = { + "id": "conn_sql", + "name": "warehouse", + "kind": "sql", + "dataset_id": ds, + "config": { + "dsn_env": "OSINT_SQL_DSN_TEST", + "query": "SELECT mmsi, name, lat, lon FROM sites ORDER BY mmsi", + "interval_s": 30, + }, + } + task = asyncio.create_task(C._run_sql(FoundryStore(get_settings()), conn)) + for _ in range(100): + await asyncio.sleep(0.05) + rows = client.get(f"/api/foundry/datasets/{ds}/rows").json()["rows"] + if len(rows) >= 3: + break + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + rows = client.get(f"/api/foundry/datasets/{ds}/rows").json()["rows"] + assert [r.get("name") for r in rows] == ["SEED", "SQL ROW ONE", "SQL ROW TWO"] + assert rows[1]["lat"] == 51.9 + + hits = client.get("/api/ontology/search", params={"q": "SQL ROW ONE"}).json() + assert any(o["props"].get("name") == "SQL ROW ONE" for o in hits), hits + + +@pytest.mark.anyio +async def test_a_query_against_a_missing_table_never_leaks_the_dsn( + source_db: str, client, monkeypatch +) -> None: + """SQLAlchemy puts a good deal into its exception text. Whatever the runner + records on the row must not include the connection string, because that row + is returned by the list route.""" + monkeypatch.setenv("OSINT_SQL_DSN_TEST", source_db) + ds = client.post( + "/api/foundry/datasets/upload", + files={"file": ("seed.csv", b"a\n1\n", "text/csv")}, + data={"name": "sql_bad"}, + ).json()["id"] + created = client.post( + "/api/foundry/connections", + json={ + "name": "bad-query", + "kind": "sql", + "dataset_id": ds, + "config": { + "dsn_env": "OSINT_SQL_DSN_TEST", + "query": "SELECT * FROM no_such_table", + "interval_s": 30, + }, + "enabled": False, + }, + ).json() + + from app.config import get_settings + from app.foundry.store import FoundryStore + + store = FoundryStore(get_settings()) + conn = {**created, "id": created["id"]} + task = asyncio.create_task(C._run_forever(conn)) + for _ in range(100): + await asyncio.sleep(0.05) + row = await store.get_connection(created["id"]) + if row and row["last_error"]: + break + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + row = await store.get_connection(created["id"]) + assert row is not None + assert row["last_error"], "the failure was never recorded on the connection" + assert "no_such_table" in row["last_error"] + assert source_db not in row["last_error"] + assert str(row["last_error"]).count("sqlite:///") == 0 + + listed = client.get("/api/foundry/connections").text + assert source_db not in listed + + +@pytest.mark.anyio +async def test_an_unset_environment_variable_is_reported_not_crashed( + client, monkeypatch +) -> None: + monkeypatch.delenv("OSINT_SQL_DSN_ABSENT", raising=False) + with pytest.raises(ValueError, match="OSINT_SQL_DSN_ABSENT is not set"): + await C._run_sql(None, {"config": {"dsn_env": "OSINT_SQL_DSN_ABSENT", "query": "SELECT 1"}}) # type: ignore[arg-type] diff --git a/apps/api/tests/test_foundry_geo_formats.py b/apps/api/tests/test_foundry_geo_formats.py new file mode 100644 index 00000000..e43170c4 --- /dev/null +++ b/apps/api/tests/test_foundry_geo_formats.py @@ -0,0 +1,225 @@ +"""Guards for the geospatial upload readers (foundry/ingest.py). + +The contract is not "we can read GeoJSON" — it is that a geospatial file lands +as rows the rest of Foundry already knows what to do with: the lat/lon sniffer +in ``foundry/geo.py`` has to find the coordinates without being told, and a +binding has to be able to mint ontology objects from the feature properties. +So the round trip, not the parse, is what is pinned here. +""" + +from __future__ import annotations + +import io +import json +import zipfile + +import pytest + +from app.foundry.geo import detect_geo, to_feature_collection +from app.foundry.ingest import parse_upload +from app.foundry.store import FoundryError + +FC = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"name": "Rotterdam", "teu": 13_400_000}, + "geometry": {"type": "Point", "coordinates": [4.4, 51.9]}, + }, + { + "type": "Feature", + "properties": {"name": "Suez"}, + "geometry": { + "type": "Polygon", + "coordinates": [[[32.0, 30.0], [32.6, 30.0], [32.6, 31.0], [32.0, 31.0]]], + }, + }, + ], +} + +KML = """ + + + Rotterdam + Europoort + 13400000 + 4.4,51.9,0 + + + Suez + + 32.0,30.0 32.6,30.0 32.6,31.0 32.0,31.0 + + + +""" + + +def _up(name: str, body: str | bytes): # type: ignore[no-untyped-def] + return parse_upload(name, body if isinstance(body, bytes) else body.encode()) + + +# ── GeoJSON ─────────────────────────────────────────────────────────────────── + + +def test_geojson_is_one_row_per_feature() -> None: + rows, _ = _up("ports.geojson", json.dumps(FC)) + assert [r["name"] for r in rows] == ["Rotterdam", "Suez"] + + +def test_geojson_point_keeps_its_exact_coordinates() -> None: + rows, _ = _up("ports.geojson", json.dumps(FC)) + assert (rows[0]["lat"], rows[0]["lon"]) == (51.9, 4.4) + + +def test_geojson_polygon_gets_its_bbox_centre() -> None: + rows, _ = _up("ports.geojson", json.dumps(FC)) + assert (rows[1]["lat"], rows[1]["lon"]) == (30.5, 32.3) + + +def test_geometry_survives_as_a_string_cell() -> None: + rows, _ = _up("ports.geojson", json.dumps(FC)) + assert json.loads(rows[1]["geometry"])["type"] == "Polygon" + assert rows[1]["geometry_type"] == "Polygon" + + +def test_feature_properties_win_over_derived_coordinates() -> None: + """A file that states its own lat/lon knows better than a bbox centre.""" + fc = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"lat": 1.0, "lon": 2.0}, + "geometry": {"type": "Point", "coordinates": [99.0, 88.0]}, + } + ], + } + rows, _ = _up("x.geojson", json.dumps(fc)) + assert (rows[0]["lat"], rows[0]["lon"]) == (1.0, 2.0) + + +def test_a_bare_feature_and_a_bare_geometry_both_load() -> None: + rows, _ = _up("one.geojson", json.dumps(FC["features"][0])) + assert rows[0]["name"] == "Rotterdam" + rows, _ = _up("g.geojson", json.dumps({"type": "Point", "coordinates": [1.0, 2.0]})) + assert (rows[0]["lat"], rows[0]["lon"]) == (2.0, 1.0) + + +def test_a_feature_with_no_geometry_is_still_a_row() -> None: + fc = { + "type": "FeatureCollection", + "features": [{"type": "Feature", "properties": {"name": "nowhere"}, "geometry": None}], + } + rows, _ = _up("x.geojson", json.dumps(fc)) + assert rows == [{"name": "nowhere"}] + + +def test_geojson_renamed_json_is_still_read_as_geojson() -> None: + rows, _ = _up("ports.json", json.dumps(FC)) + assert rows[0]["lat"] == 51.9 + + +def test_a_plain_json_array_is_unaffected() -> None: + rows, _ = _up("rows.json", json.dumps([{"a": 1}, {"a": 2}])) + assert rows == [{"a": 1}, {"a": 2}] + + +def test_a_json_object_that_is_not_geojson_still_reports_the_array_error() -> None: + with pytest.raises(FoundryError) as exc: + _up("x.json", json.dumps({"not": "geojson"})) + assert "array of objects" in str(exc.value.detail) + + +def test_geojson_without_a_type_is_a_422() -> None: + with pytest.raises(FoundryError) as exc: + _up("x.geojson", json.dumps({"features": []})) + assert exc.value.status_code == 422 + + +# ── KML / KMZ ───────────────────────────────────────────────────────────────── + + +def test_kml_placemarks_become_rows() -> None: + rows, _ = _up("ports.kml", KML) + assert [r["name"] for r in rows] == ["Rotterdam", "Suez"] + assert rows[0]["description"] == "Europoort" + + +def test_kml_point_coordinates_are_lon_lat_alt() -> None: + """KML orders a coordinate lon,lat — reading it as lat,lon puts every pin in + the wrong hemisphere, which is the classic way to get this wrong.""" + rows, _ = _up("ports.kml", KML) + assert (rows[0]["lat"], rows[0]["lon"]) == (51.9, 4.4) + + +def test_kml_polygon_keeps_its_ring() -> None: + rows, _ = _up("ports.kml", KML) + geom = json.loads(rows[1]["geometry"]) + assert geom["type"] == "Polygon" + assert len(geom["coordinates"][0]) == 4 + + +def test_kml_extended_data_becomes_a_typed_column() -> None: + rows, schema = _up("ports.kml", KML) + assert rows[0]["teu"] == 13_400_000 + assert {c["name"]: c["type"] for c in schema}["teu"] == "int" + + +def test_kml_without_a_namespace_reads_the_same() -> None: + bare = KML.replace(' xmlns="http://www.opengis.net/kml/2.2"', "") + assert [r["name"] for r in _up("x.kml", bare)[0]] == ["Rotterdam", "Suez"] + + +def test_kml_2_1_namespace_reads_the_same() -> None: + old = KML.replace("kml/2.2", "kml/2.1") + assert [r["name"] for r in _up("x.kml", old)[0]] == ["Rotterdam", "Suez"] + + +def test_malformed_kml_is_a_422_not_a_500() -> None: + with pytest.raises(FoundryError) as exc: + _up("x.kml", "") + assert exc.value.status_code == 422 + + +def test_kmz_reads_its_inner_kml() -> None: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("images/logo.png", b"not kml") + zf.writestr("doc.kml", KML) + rows, _ = _up("ports.kmz", buf.getvalue()) + assert [r["name"] for r in rows] == ["Rotterdam", "Suez"] + + +def test_kmz_without_a_kml_is_a_422() -> None: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("readme.txt", b"nope") + with pytest.raises(FoundryError) as exc: + _up("x.kmz", buf.getvalue()) + assert exc.value.status_code == 422 + + +def test_a_kmz_that_is_not_a_zip_is_a_422() -> None: + with pytest.raises(FoundryError) as exc: + _up("x.kmz", b"definitely not a zip") + assert exc.value.status_code == 422 + + +# ── the round trip that is the actual point ─────────────────────────────────── + + +@pytest.mark.parametrize("name,body", [("p.geojson", json.dumps(FC)), ("p.kml", KML)]) +def test_an_uploaded_map_file_comes_back_out_of_the_geo_route(name: str, body: str) -> None: + """detect_geo has to find the coordinates unaided, because nothing in the + upload path tells it which columns they are.""" + rows, schema = _up(name, body) + cols = detect_geo(schema, rows) + assert cols == {"lat_col": "lat", "lon_col": "lon"} + fc = to_feature_collection(rows, **{"lat_col": cols["lat_col"], "lon_col": cols["lon_col"]}) + assert [f["geometry"]["coordinates"] for f in fc["features"]] == [ + [4.4, 51.9], + [32.3, 30.5], + ] + assert fc["features"][0]["properties"]["name"] == "Rotterdam" diff --git a/apps/api/tests/test_ingest_webhook.py b/apps/api/tests/test_ingest_webhook.py new file mode 100644 index 00000000..31631e8d --- /dev/null +++ b/apps/api/tests/test_ingest_webhook.py @@ -0,0 +1,251 @@ +"""Guards for the inbound push endpoint (routes/ingest.py). + +Every other way data enters this platform is a pull the platform initiated. This +one route is the exception, so it is the one place an unauthenticated stranger +can write, and the tests here are mostly about that: the token is required, it +is compared in constant time, it is never handed back or logged, and a body that +is too large is refused before it is parsed rather than after. + +The functional half is the reason the route is only a few lines: a pushed row +goes through the SAME append + auto-sync path an upload does, so it lands in the +ontology through whatever binding the operator already configured. +""" + +from __future__ import annotations + +import json + +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture +def dataset(client: TestClient) -> str: + r = client.post( + "/api/foundry/datasets/upload", + files={"file": ("seed.csv", b"mmsi,name\n1,Alpha\n", "text/csv")}, + data={"name": "pushed"}, + ) + assert r.status_code == 200, r.text + return r.json()["dataset_id"] if "dataset_id" in r.json() else r.json()["id"] + + +def _mint(client: TestClient, dataset_id: str) -> str: + r = client.post(f"/api/foundry/datasets/{dataset_id}/ingest-token") + assert r.status_code == 200, r.text + return r.json()["token"] + + +# ── the token is a credential ───────────────────────────────────────────────── + + +def test_no_token_is_rejected(client: TestClient, dataset: str) -> None: + _mint(client, dataset) + r = client.post(f"/api/ingest/{dataset}", json={"mmsi": 2}) + assert r.status_code == 401 + + +def test_a_wrong_token_is_rejected(client: TestClient, dataset: str) -> None: + _mint(client, dataset) + r = client.post( + f"/api/ingest/{dataset}", + json={"mmsi": 2}, + headers={"X-Ingest-Token": "not-the-token"}, + ) + assert r.status_code == 401 + + +def test_a_dataset_with_no_token_has_no_ingest_endpoint( + client: TestClient, dataset: str +) -> None: + r = client.post( + f"/api/ingest/{dataset}", json={"mmsi": 2}, headers={"X-Ingest-Token": "x"} + ) + assert r.status_code == 404 + + +def test_an_unknown_dataset_answers_the_same_as_an_unarmed_one( + client: TestClient, dataset: str +) -> None: + """Same status and same detail, so the endpoint cannot be used to find out + which dataset ids exist.""" + unarmed = client.post( + f"/api/ingest/{dataset}", json={}, headers={"X-Ingest-Token": "x"} + ) + unknown = client.post( + "/api/ingest/ds_doesnotexist", json={}, headers={"X-Ingest-Token": "x"} + ) + assert unarmed.status_code == unknown.status_code == 404 + assert unarmed.json()["detail"] == unknown.json()["detail"] + + +def test_the_token_is_shown_once_and_never_again( + client: TestClient, dataset: str +) -> None: + token = _mint(client, dataset) + listed = client.get("/api/foundry/datasets").json() + assert token not in json.dumps(listed) + one = client.get(f"/api/foundry/datasets/{dataset}").json() + assert token not in json.dumps(one) + assert "ingest_token" not in json.dumps(one) + + +def test_reminting_replaces_the_previous_token( + client: TestClient, dataset: str +) -> None: + old = _mint(client, dataset) + new = _mint(client, dataset) + assert old != new + assert ( + client.post( + f"/api/ingest/{dataset}", json={"mmsi": 9}, headers={"X-Ingest-Token": old} + ).status_code + == 401 + ) + assert ( + client.post( + f"/api/ingest/{dataset}", json={"mmsi": 9}, headers={"X-Ingest-Token": new} + ).status_code + == 200 + ) + + +def test_revoking_closes_the_endpoint(client: TestClient, dataset: str) -> None: + token = _mint(client, dataset) + assert client.delete(f"/api/foundry/datasets/{dataset}/ingest-token").status_code == 200 + r = client.post( + f"/api/ingest/{dataset}", json={"mmsi": 3}, headers={"X-Ingest-Token": token} + ) + assert r.status_code == 404 + + +# ── what a push does ────────────────────────────────────────────────────────── + + +def test_one_object_appends_one_row(client: TestClient, dataset: str) -> None: + token = _mint(client, dataset) + r = client.post( + f"/api/ingest/{dataset}", + json={"mmsi": 2, "name": "Bravo"}, + headers={"X-Ingest-Token": token}, + ) + assert r.status_code == 200 + assert r.json()["rows_added"] == 1 + rows = client.get(f"/api/foundry/datasets/{dataset}/rows").json()["rows"] + assert [row["name"] for row in rows] == ["Alpha", "Bravo"] + + +def test_an_array_appends_every_row(client: TestClient, dataset: str) -> None: + token = _mint(client, dataset) + r = client.post( + f"/api/ingest/{dataset}", + json=[{"mmsi": 2, "name": "Bravo"}, {"mmsi": 3, "name": "Charlie"}], + headers={"X-Ingest-Token": token}, + ) + assert r.json()["rows_added"] == 2 + rows = client.get(f"/api/foundry/datasets/{dataset}/rows").json()["rows"] + assert len(rows) == 3 + + +def test_a_push_reaches_the_ontology_through_an_existing_binding( + client: TestClient, dataset: str +) -> None: + """The whole point of reusing append + auto_sync rather than writing a new + ingest path: a pushed row becomes an ontology object with no extra wiring.""" + b = client.post( + "/api/foundry/bindings", + json={ + "dataset_id": dataset, + "object_kind": "vessel", + "key_column": "mmsi", + "prop_map": {"name": "name"}, + }, + ) + assert b.status_code == 200, b.text + token = _mint(client, dataset) + client.post( + f"/api/ingest/{dataset}", + json={"mmsi": 636092000, "name": "EVER GIVEN"}, + headers={"X-Ingest-Token": token}, + ) + hits = client.get("/api/ontology/search", params={"q": "EVER GIVEN"}).json() + assert any(o["props"].get("name") == "EVER GIVEN" for o in hits), hits + + +def test_an_empty_array_is_a_no_op_not_an_error( + client: TestClient, dataset: str +) -> None: + token = _mint(client, dataset) + r = client.post( + f"/api/ingest/{dataset}", json=[], headers={"X-Ingest-Token": token} + ) + assert r.status_code == 200 + assert r.json()["rows_added"] == 0 + + +def test_a_non_object_body_is_a_422(client: TestClient, dataset: str) -> None: + token = _mint(client, dataset) + for body in ("[1, 2, 3]", '"a string"', "42"): + r = client.post( + f"/api/ingest/{dataset}", + content=body, + headers={"X-Ingest-Token": token, "content-type": "application/json"}, + ) + assert r.status_code == 422, body + + +def test_malformed_json_is_a_422(client: TestClient, dataset: str) -> None: + token = _mint(client, dataset) + r = client.post( + f"/api/ingest/{dataset}", + content="{not json", + headers={"X-Ingest-Token": token, "content-type": "application/json"}, + ) + assert r.status_code == 422 + + +def test_an_oversized_body_is_refused_before_it_is_parsed( + client: TestClient, dataset: str +) -> None: + """A 413 that arrives only after json.loads has already built the object in + memory is not a size cap.""" + from app.foundry.store import MAX_UPLOAD_BYTES + + token = _mint(client, dataset) + body = b'[{"pad": "' + b"x" * (MAX_UPLOAD_BYTES + 1024) + b'"}]' + r = client.post( + f"/api/ingest/{dataset}", + content=body, + headers={"X-Ingest-Token": token, "content-type": "application/json"}, + ) + assert r.status_code == 413 + + +def test_an_oversized_chunked_body_is_also_refused( + client: TestClient, dataset: str +) -> None: + """The Content-Length branch is the easy half. A chunked upload declares no + length at all, so only the running total in the stream loop stops it.""" + from app.foundry.store import MAX_UPLOAD_BYTES + + token = _mint(client, dataset) + + def _chunks(): # type: ignore[no-untyped-def] + yield b'[{"pad": "' + for _ in range((MAX_UPLOAD_BYTES // 65536) + 2): + yield b"x" * 65536 + yield b'"}]' + + r = client.post( + f"/api/ingest/{dataset}", + content=_chunks(), + headers={"X-Ingest-Token": token, "content-type": "application/json"}, + ) + assert "content-length" not in {k.lower() for k in r.request.headers} + assert r.status_code == 413 + + +def test_minting_a_token_for_an_unknown_dataset_is_a_404(client: TestClient) -> None: + assert ( + client.post("/api/foundry/datasets/ds_nope/ingest-token").status_code == 404 + ) diff --git a/apps/api/tests/test_mqtt_client.py b/apps/api/tests/test_mqtt_client.py new file mode 100644 index 00000000..9d6bd2d0 --- /dev/null +++ b/apps/api/tests/test_mqtt_client.py @@ -0,0 +1,220 @@ +"""The MQTT subscriber, against a real socket. + +``test_connections.py`` covers configuration, availability and supervision, and +``test_ais_keyless.py`` covers the wire codec as pure functions. Neither of them +ever opens a connection, so the part that actually talks to a broker — CONNECT, +wait for CONNACK, SUBSCRIBE, decode PUBLISH, keep the session alive — was the +one piece of the MQTT connection with no evidence behind it. + +The broker here is forty lines of asyncio speaking MQTT 3.1.1 back. That is the +whole point: a public broker would make this test a network probe that fails on +a laptop with no egress, and a mock of our own client would prove nothing about +the bytes. +""" + +from __future__ import annotations + +import asyncio +import json + +import pytest + +from app import mqtt_client +from app.foundry import connections as C + + +class FakeBroker: + """Accepts one client, answers CONNACK and SUBACK, then publishes.""" + + def __init__(self, connack_code: int = 0) -> None: + self.connack_code = connack_code + self.port = 0 + self.subscribed_topic: str | None = None + self.pings = 0 + self._server: asyncio.AbstractServer | None = None + self._to_publish: list[tuple[str, bytes]] = [] + self._ready = asyncio.Event() + + def publish_later(self, topic: str, payload: bytes) -> None: + self._to_publish.append((topic, payload)) + + async def start(self) -> None: + self._server = await asyncio.start_server(self._serve, "127.0.0.1", 0) + self.port = self._server.sockets[0].getsockname()[1] + + async def stop(self) -> None: + if self._server is not None: + self._server.close() + await self._server.wait_closed() + + @staticmethod + def _publish_packet(topic: str, payload: bytes) -> bytes: + body = len(topic).to_bytes(2, "big") + topic.encode() + payload + return b"\x30" + mqtt_client.enc_remaining_length(len(body)) + body + + async def _serve( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + buf = b"" + try: + while True: + chunk = await reader.read(65536) + if not chunk: + return + buf += chunk + packets, buf = mqtt_client.parse_packets(buf) + for ptype, _b0, body in packets: + if ptype == 1: # CONNECT + writer.write(b"\x20\x02\x00" + bytes([self.connack_code])) + await writer.drain() + elif ptype == 8: # SUBSCRIBE + tlen = int.from_bytes(body[2:4], "big") + self.subscribed_topic = body[4 : 4 + tlen].decode() + packet_id = body[0:2] + writer.write(b"\x90\x03" + packet_id + b"\x00") + for topic, payload in self._to_publish: + writer.write(self._publish_packet(topic, payload)) + await writer.drain() + self._ready.set() + elif ptype == 12: # PINGREQ + self.pings += 1 + writer.write(b"\xd0\x00") + await writer.drain() + except (ConnectionError, asyncio.CancelledError): + return + finally: + writer.close() + + +@pytest.fixture +async def broker(): # type: ignore[no-untyped-def] + b = FakeBroker() + await b.start() + try: + yield b + finally: + await b.stop() + + +async def _collect(url: str, topic: str, n: int, wait_s: float = 5.0): # type: ignore[no-untyped-def] + got: list[tuple[str, bytes]] = [] + + async def _run() -> None: + async for msg in mqtt_client.subscribe(url, topic): + got.append(msg) + if len(got) >= n: + return + + await asyncio.wait_for(_run(), timeout=wait_s) + return got + + +@pytest.mark.anyio +async def test_subscribes_and_receives_over_tcp(broker: FakeBroker) -> None: + broker.publish_later("sensors/1", b'{"t": 21.5}') + got = await _collect(f"mqtt://127.0.0.1:{broker.port}", "sensors/#", 1) + assert got == [("sensors/1", b'{"t": 21.5}')] + assert broker.subscribed_topic == "sensors/#" + + +@pytest.mark.anyio +async def test_receives_several_messages_in_order(broker: FakeBroker) -> None: + for i in range(3): + broker.publish_later(f"sensors/{i}", str(i).encode()) + got = await _collect(f"mqtt://127.0.0.1:{broker.port}", "sensors/#", 3) + assert [t for t, _ in got] == ["sensors/0", "sensors/1", "sensors/2"] + + +@pytest.mark.anyio +async def test_a_refused_connection_raises_rather_than_hanging(broker: FakeBroker) -> None: + """A broker that says no must surface, not sit in the read loop forever — + the runner's backoff can only work if the failure reaches it.""" + broker.connack_code = 5 # not authorised + with pytest.raises(ConnectionError, match="refused"): + await _collect(f"mqtt://127.0.0.1:{broker.port}", "x", 1, wait_s=5.0) + + +@pytest.mark.anyio +async def test_an_unreachable_broker_raises() -> None: + with pytest.raises((ConnectionError, OSError)): + await _collect("mqtt://127.0.0.1:1", "x", 1, wait_s=5.0) + + +@pytest.mark.parametrize( + "url,err", + [("http://h/x", "unsupported"), ("mqtt://", "no host"), ("nonsense", "unsupported")], +) +@pytest.mark.anyio +async def test_a_bad_url_is_rejected_before_dialling(url: str, err: str) -> None: + with pytest.raises(ValueError, match=err): + await _collect(url, "x", 1, wait_s=5.0) + + +@pytest.mark.anyio +async def test_the_default_port_is_1883() -> None: + """A url with no port must not dial port 0.""" + with pytest.raises((ConnectionError, OSError)): + await _collect("mqtt://127.0.0.1", "x", 1, wait_s=5.0) + + +# ── the connection runner, end to end over the same socket ──────────────────── + + +@pytest.mark.anyio +async def test_an_mqtt_connection_lands_rows_and_mints_ontology_objects( + broker: FakeBroker, client +) -> None: # type: ignore[no-untyped-def] + """The whole path with nothing mocked but the broker: a published message + becomes a dataset row and then an ontology object through a binding.""" + ds = client.post( + "/api/foundry/datasets/upload", + files={"file": ("seed.csv", b"mmsi,name\n1,SEED\n", "text/csv")}, + data={"name": "mqtt_target"}, + ).json()["id"] + assert client.post( + "/api/foundry/bindings", + json={ + "dataset_id": ds, + "object_kind": "vessel", + "key_column": "mmsi", + "prop_map": {"name": "name"}, + }, + ).status_code == 200 + + # Two messages, so the batch flushes on the deadline rather than the count. + broker.publish_later("vessels/1", json.dumps({"mmsi": 636092111, "name": "FROM MQTT"}).encode()) + broker.publish_later("vessels/2", json.dumps({"mmsi": 636092222, "name": "ALSO MQTT"}).encode()) + + conn = { + "id": "conn_test", + "name": "t", + "kind": "mqtt", + "dataset_id": ds, + "config": {"url": f"mqtt://127.0.0.1:{broker.port}", "topic": "vessels/#"}, + } + from app.config import get_settings + from app.foundry.store import FoundryStore + + store = FoundryStore(get_settings()) + # Flush as soon as both messages have arrived instead of waiting out the + # 10 s deadline; the batching rule itself is covered by its own assertion. + original = C._BATCH_AGE_S + C._BATCH_AGE_S = 0.0 + try: + task = asyncio.create_task(C._run_mqtt(store, conn)) + for _ in range(100): + await asyncio.sleep(0.05) + rows = client.get(f"/api/foundry/datasets/{ds}/rows").json()["rows"] + if len(rows) >= 3: + break + task.cancel() + await asyncio.gather(task, return_exceptions=True) + finally: + C._BATCH_AGE_S = original + + rows = client.get(f"/api/foundry/datasets/{ds}/rows").json()["rows"] + assert [r.get("name") for r in rows] == ["SEED", "FROM MQTT", "ALSO MQTT"] + assert rows[1]["_topic"] == "vessels/1" + + hits = client.get("/api/ontology/search", params={"q": "FROM MQTT"}).json() + assert any(o["props"].get("name") == "FROM MQTT" for o in hits), hits diff --git a/apps/api/tests/test_ontology_schema.py b/apps/api/tests/test_ontology_schema.py new file mode 100644 index 00000000..a5241e93 --- /dev/null +++ b/apps/api/tests/test_ontology_schema.py @@ -0,0 +1,208 @@ +"""Guards for the declared ontology schema (intel/ontology_schema.py). + +Two operator decisions are enforced here: + +1. **The vocabulary has one source of truth.** ``KNOWN_RELS`` is derived from + ``REL_TYPES``; before this it was a second hand-kept frozenset and thirteen + relations the code was actually minting had fallen out of it. A verb minted + anywhere in ``apps/api`` must be declared. +2. **Validation warns, it never rejects.** The registry has to be able to hold + an edge an analyst or a language model invents + (``intel/ontology.py``'s KNOWN_RELS comment, ``routes/extract.py``'s + model-authored rels). Every validator therefore has to survive garbage. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from app.intel.ontology import KNOWN_RELS +from app.intel.ontology_schema import ( + PROP_TYPES, + REL_TYPES, + label_for, + schema_payload, + validate_link, + validate_object, +) + +_APP_DIR = Path(__file__).resolve().parents[1] / "app" + + +def test_known_rels_is_derived_from_rel_types() -> None: + assert KNOWN_RELS == frozenset(REL_TYPES) + + +def test_every_relation_names_both_ends() -> None: + for rel, rt in REL_TYPES.items(): + assert rt.forward, f"{rel} has no forward label" + assert rt.inverse, f"{rel} has no inverse label" + + +def test_labels_carry_no_em_dash() -> None: + """Relation labels render in the dashboard, so the copy rule binds them + (apps/web/CLAUDE.md, docs/decisions.md#dashboard-copy-one-voice…).""" + for rel, rt in REL_TYPES.items(): + assert "—" not in rt.forward, rel + assert "—" not in rt.inverse, rel + + +def test_symmetric_relations_read_the_same_from_both_ends() -> None: + for rel in ("correlated", "peers_with", "same_as"): + rt = REL_TYPES[rel] + assert rt.forward == rt.inverse, rel + + +def test_contains_and_part_of_are_each_other() -> None: + assert REL_TYPES["contains"].forward == REL_TYPES["part_of"].inverse + assert REL_TYPES["contains"].inverse == REL_TYPES["part_of"].forward + + +def test_every_rel_minted_in_the_backend_is_declared() -> None: + """The drift guard. Any ``rel="…"`` or ``.link(src, dst, "…")`` literal under + ``apps/api/app`` has to appear in REL_TYPES, or the schema is lying again.""" + minted: set[str] = set() + for path in _APP_DIR.rglob("*.py"): + text = path.read_text(encoding="utf-8") + minted.update(re.findall(r'\brel="([a-z_]+)"', text)) + minted.update(re.findall(r'\.link\([^()]*"([a-z_]+)"\)', text)) + # `enclosure` is an RSS in news/sources.py, not an ontology edge. + minted.discard("enclosure") + assert minted, "the scan found no rel literals at all — the regex broke" + assert minted <= set(REL_TYPES), sorted(minted - set(REL_TYPES)) + + +@pytest.mark.parametrize( + "rel,reverse,expected", + [ + ("officer_of", False, "officer of"), + ("officer_of", True, "has officer"), + ("not_a_real_rel", False, "not a real rel"), + ("not_a_real_rel", True, "not a real rel"), + ], +) +def test_label_for(rel: str, reverse: bool, expected: str) -> None: + assert label_for(rel, reverse=reverse) == expected + + +# ── validation never rejects ────────────────────────────────────────────────── + + +def test_validate_object_is_silent_on_an_undeclared_kind() -> None: + assert validate_object("no_such_kind", {"anything": object()}) == [] + + +def test_validate_object_is_silent_on_an_undeclared_prop() -> None: + assert validate_object("aircraft", {"invented_by_an_agent": 3}) == [] + + +def test_validate_object_accepts_a_missing_prop() -> None: + assert validate_object("aircraft", {}) == [] + + +def test_validate_object_ignores_none() -> None: + """A feed that reports no value writes None; that is the never-guess rule, + not a type error.""" + assert validate_object("aircraft", {"velocity_ms": None}) == [] + + +def test_validate_object_flags_a_contradicted_type() -> None: + warnings = validate_object("aircraft", {"velocity_ms": "fast"}) + assert len(warnings) == 1 + assert "velocity_ms" in warnings[0] + + +def test_bool_is_not_a_number() -> None: + """bool subclasses int, so a naive isinstance check would pass True as a + velocity and reject a real bool prop.""" + assert validate_object("aircraft", {"velocity_ms": True}) + assert validate_object("aircraft", {"on_ground": True}) == [] + assert validate_object("aircraft", {"on_ground": 1}) + + +def test_mmsi_may_be_an_int_and_icao24_a_string() -> None: + assert validate_object("vessel", {"mmsi": 636092000}) == [] + assert validate_object("aircraft", {"icao24": "4ca7b3"}) == [] + + +def test_validate_link_warns_on_an_unknown_rel_but_returns() -> None: + warnings = validate_link("model_invented_this", "domain", "ip") + assert len(warnings) == 1 + assert "not a declared relation" in warnings[0] + + +def test_validate_link_accepts_a_declared_pair() -> None: + assert validate_link("resolves_to", "domain", "ip") == [] + + +def test_validate_link_flags_a_wrong_endpoint() -> None: + warnings = validate_link("resolves_to", "vessel", "ip") + assert len(warnings) == 1 + assert "source" in warnings[0] + + +def test_validate_link_unconstrained_endpoints_never_warn() -> None: + assert validate_link("same_as", "aircraft", "wallet") == [] + + +def test_validators_survive_hostile_input() -> None: + """The point of warn-never-reject: nothing in here may raise.""" + assert validate_object("aircraft", {"callsign": {"nested": [1, 2]}}) + validate_object("", {}) + validate_link("", "", "") + label_for("") + + +# ── the payload the frontend reads ──────────────────────────────────────────── + + +def test_schema_payload_shape() -> None: + payload = schema_payload() + assert set(payload) == {"relations", "kinds"} + assert set(payload["relations"]) == set(REL_TYPES) + assert set(payload["kinds"]) == set(PROP_TYPES) + officer = payload["relations"]["officer_of"] + assert officer == { + "forward": "officer of", + "inverse": "has officer", + "src_kinds": ["person"], + "dst_kinds": ["org"], + } + + +def test_schema_route_is_keyless(client: TestClient) -> None: + r = client.get("/api/ontology/schema") + assert r.status_code == 200 + assert r.json()["relations"]["contains"]["inverse"] == "part of" + + +def test_object_post_reports_warnings_without_refusing_the_write( + client: TestClient, +) -> None: + r = client.post( + "/api/ontology/object", + json={"id": "aircraft:testaa", "props": {"velocity_ms": "fast"}}, + ) + assert r.status_code == 200 + body = r.json() + # The object is stored verbatim — warnings describe it, they do not gate it. + assert body["id"] == "aircraft:testaa" + assert body["props"] == {"velocity_ms": "fast"} + assert len(body["warnings"]) == 1 + + stored = client.get("/api/ontology/object/aircraft:testaa") + assert stored.status_code == 200 + assert stored.json()["props"] == {"velocity_ms": "fast"} + + +def test_object_post_is_quiet_when_nothing_is_wrong(client: TestClient) -> None: + r = client.post( + "/api/ontology/object", + json={"id": "aircraft:testbb", "props": {"callsign": "TEST123"}}, + ) + assert r.status_code == 200 + assert r.json()["warnings"] == [] diff --git a/apps/api/tests/test_ontology_search.py b/apps/api/tests/test_ontology_search.py new file mode 100644 index 00000000..c09403c9 --- /dev/null +++ b/apps/api/tests/test_ontology_search.py @@ -0,0 +1,201 @@ +"""Guards for full-text search over the ontology (SqliteRegistry.search). + +Before this the graph could only be reached by exact canonical id, so the +behaviour worth pinning is not "search works" but the three ways it silently +would not: the index drifting out of step with a write, hostile input reaching +FTS5 as syntax, and an existing deployment whose rows predate the index. +""" + +from __future__ import annotations + +import json +import sqlite3 + +import pytest +from fastapi.testclient import TestClient + +from app.intel import ontology_local +from app.intel.ontology import Object, get_registry +from app.keys import UserCtx + + +@pytest.fixture +def reg(tmp_path): # type: ignore[no-untyped-def] + ontology_local.override_db_path(str(tmp_path / "ont.db")) + try: + yield get_registry(UserCtx(user_id="u1", token=None)) + finally: + ontology_local.override_db_path(None) + + +async def _mint(reg, obj_id: str, props: dict) -> None: # type: ignore[no-untyped-def] + await reg.upsert(Object(id=obj_id, props=props)) + + +@pytest.mark.anyio +async def test_finds_an_object_by_a_property_value(reg) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "aircraft:4ca7b3", {"callsign": "RYR1234"}) + hits = await reg.search("RYR1234") + assert [o.id for o in hits] == ["aircraft:4ca7b3"] + + +@pytest.mark.anyio +async def test_finds_an_object_by_a_property_name(reg) -> None: # type: ignore[no-untyped-def] + """The Explorer facet case: which objects report a callsign at all.""" + await _mint(reg, "aircraft:4ca7b3", {"callsign": "RYR1234"}) + await _mint(reg, "vessel:636092000", {"name": "EVER GIVEN"}) + assert [o.id for o in await reg.search("callsign")] == ["aircraft:4ca7b3"] + + +@pytest.mark.anyio +async def test_finds_an_object_by_its_id_without_the_prefix(reg) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "vessel:636092000", {}) + assert [o.id for o in await reg.search("636092000")] == ["vessel:636092000"] + + +@pytest.mark.anyio +async def test_prefix_match(reg) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "vessel:636092000", {"name": "EVER GIVEN"}) + assert [o.id for o in await reg.search("EVERG")] == [] + assert [o.id for o in await reg.search("EVER")] == ["vessel:636092000"] + + +@pytest.mark.anyio +async def test_nested_values_are_searchable(reg) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "incident:abc", {"domains": ["maritime", "aviation"]}) + assert [o.id for o in await reg.search("maritime")] == ["incident:abc"] + + +@pytest.mark.anyio +async def test_kind_filter(reg) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "aircraft:4ca7b3", {"name": "SHARED"}) + await _mint(reg, "vessel:636092000", {"name": "SHARED"}) + hits = await reg.search("SHARED", kinds=["vessel"]) + assert [o.id for o in hits] == ["vessel:636092000"] + + +@pytest.mark.anyio +async def test_index_follows_an_update(reg) -> None: # type: ignore[no-untyped-def] + """upsert replaces props wholesale, so the OLD value must stop matching.""" + await _mint(reg, "aircraft:4ca7b3", {"callsign": "OLDCALL"}) + await _mint(reg, "aircraft:4ca7b3", {"callsign": "NEWCALL"}) + assert await reg.search("OLDCALL") == [] + assert [o.id for o in await reg.search("NEWCALL")] == ["aircraft:4ca7b3"] + + +@pytest.mark.anyio +async def test_index_follows_assert_props(reg) -> None: # type: ignore[no-untyped-def] + """assert_props MERGES, so both the old and the new value stay findable.""" + await _mint(reg, "aircraft:4ca7b3", {"callsign": "RYR1234"}) + await reg.assert_props("aircraft:4ca7b3", {"registration": "EIDPZ"}, source="t") + assert [o.id for o in await reg.search("EIDPZ")] == ["aircraft:4ca7b3"] + assert [o.id for o in await reg.search("RYR1234")] == ["aircraft:4ca7b3"] + + +@pytest.mark.anyio +async def test_index_follows_a_delete(reg) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "aircraft:4ca7b3", {"callsign": "RYR1234"}) + await reg.delete("aircraft:4ca7b3") + assert await reg.search("RYR1234") == [] + + +@pytest.mark.anyio +async def test_search_is_scoped_to_the_caller(reg, tmp_path) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "aircraft:4ca7b3", {"callsign": "PRIVATE"}) + other = get_registry(UserCtx(user_id="u2", token=None)) + assert await other.search("PRIVATE") == [] + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "q", + [ + 'unbalanced "quote', + "AAL-123", + "NEAR(a b)", + "col:umn", + "*", + "^caret", + " ", + "()", + "a AND OR b", + ], +) +async def test_hostile_queries_never_raise(reg, q: str) -> None: # type: ignore[no-untyped-def] + """FTS5 treats quotes, hyphens, colons, carets and NEAR as syntax. A search + box must answer, not 500.""" + await _mint(reg, "aircraft:4ca7b3", {"callsign": "RYR1234"}) + assert isinstance(await reg.search(q), list) + + +@pytest.mark.anyio +async def test_hyphenated_input_still_matches(reg) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "aircraft:4ca7b3", {"callsign": "AAL123"}) + assert [o.id for o in await reg.search("AAL123-")] == ["aircraft:4ca7b3"] + + +@pytest.mark.anyio +async def test_backfills_rows_written_before_the_index_existed(reg, tmp_path) -> None: # type: ignore[no-untyped-def] + """Simulates an upgraded deployment: object rows present, index empty.""" + await _mint(reg, "aircraft:4ca7b3", {"callsign": "LEGACY1"}) + con = sqlite3.connect(str(tmp_path / "ont.db")) + con.execute("DELETE FROM objects_fts") + con.commit() + con.close() + ontology_local._fts_backfilled.clear() + + assert [o.id for o in await reg.search("LEGACY1")] == ["aircraft:4ca7b3"] + + +@pytest.mark.anyio +async def test_backfill_keeps_each_user_separate(reg, tmp_path) -> None: # type: ignore[no-untyped-def] + await _mint(reg, "aircraft:4ca7b3", {"callsign": "MINE"}) + other = get_registry(UserCtx(user_id="u2", token=None)) + await other.upsert(Object(id="aircraft:beef01", props={"callsign": "THEIRS"})) + con = sqlite3.connect(str(tmp_path / "ont.db")) + con.execute("DELETE FROM objects_fts") + con.commit() + con.close() + ontology_local._fts_backfilled.clear() + + assert [o.id for o in await reg.search("MINE")] == ["aircraft:4ca7b3"] + assert await reg.search("THEIRS") == [] + + +@pytest.mark.anyio +async def test_a_large_blob_cannot_dominate_the_index(reg) -> None: # type: ignore[no-untyped-def] + """One prop is truncated, so a saved investigation's node list does not put + a megabyte of ids into the index.""" + await _mint(reg, "investigation:big", {"nodes": ["x" * 50_000]}) + con = sqlite3.connect(ontology_local._resolved_db_path()) + (text,) = con.execute("SELECT text FROM objects_fts").fetchone() + con.close() + assert len(text) < 1_000 + + +def test_route_finds_a_promoted_object(client: TestClient) -> None: + client.post( + "/api/ontology/object", + json={"id": "vessel:636092099", "props": {"name": "SEARCHABLE"}}, + ) + r = client.get("/api/ontology/search", params={"q": "SEARCHABLE"}) + assert r.status_code == 200 + assert [o["id"] for o in r.json()] == ["vessel:636092099"] + + +def test_route_rejects_an_empty_query(client: TestClient) -> None: + assert client.get("/api/ontology/search", params={"q": ""}).status_code == 422 + + +def test_route_answers_empty_for_punctuation_only(client: TestClient) -> None: + r = client.get("/api/ontology/search", params={"q": "***"}) + assert r.status_code == 200 + assert r.json() == [] + + +def test_props_survive_a_round_trip_through_the_index(client: TestClient) -> None: + """The index must not touch the props blob the frontend round-trips.""" + props = {"name": "EVER GIVEN", "nested": {"a": [1, 2]}, "none": None} + client.post("/api/ontology/object", json={"id": "vessel:9811000", "props": props}) + got = client.get("/api/ontology/object/vessel:9811000").json() + assert got["props"] == json.loads(json.dumps(props)) diff --git a/apps/web/src/AppRouter.tsx b/apps/web/src/AppRouter.tsx index 430bd587..0a3254c2 100644 --- a/apps/web/src/AppRouter.tsx +++ b/apps/web/src/AppRouter.tsx @@ -4,7 +4,6 @@ import { App } from './App.js'; import { AuthProvider, useAuth } from './auth/AuthContext.js'; import { SettingsModal } from './settings/SettingsModal.js'; import { useSettings } from './state/settings.js'; -import { useAppView, APP_META } from './state/appView.js'; import { Onboarding, hasOnboarded } from './onboarding/Onboarding.js'; import { isSupabaseConfigured } from './transport/supabase.js'; import { AiSetupWizard } from './settings/localAi/AiSetupWizard.js'; @@ -129,24 +128,19 @@ function TopBar(): JSX.Element | null { function PredictedMotionBadge(): JSX.Element | null { const loc = useLocation(); const on = useSettings((s) => s.aircraftDeadReckon); - const activeApp = useAppView((s) => s.app); if (!on) return null; - if (loc.pathname !== '/' && !loc.pathname.startsWith('/2d')) return null; - // The badge annotates aircraft on the globe. A full-surface app (AI, Foundry, - // Workflows, City, Country) covers the globe, so the badge is meaningless - // there — and worse, the app surface clips its 427px width down to a stray - // "● Pr" at the left edge. Hide it whenever the globe isn't the active view. - if (loc.pathname === '/' && APP_META[activeApp].chrome === 'full') return null; - // The console home ("/") stacks a ~158px timeline footer AND the AgentConsole's - // resting slash-hints row above it; sit clear of BOTH so the badge never lands - // in the console's text band. The 2D route has a clear bottom. - const bottomClass = loc.pathname === '/' ? 'bottom-[280px]' : 'bottom-2'; - // Centered so it never sits under the left tool flyout (Layers/Feeds/etc.), - // which overlays the map's bottom-left corner where this used to pin. + // On the console globe ("/") this marker lives in the GlobeOverlays + // bottom-right status cluster with the other map readouts. Only the 2D + // route, which has no status footer, gets a floating chip — bottom-right, + // clear of the top-right Settings/3D/2D cluster. + if (!loc.pathname.startsWith('/2d')) return null; return ( -
+
- Predicted motion: aircraft positions estimated between ADS-B fixes + motion predicted · estimated between ADS-B fixes
); } diff --git a/apps/web/src/explorer/ExplorerApp.tsx b/apps/web/src/explorer/ExplorerApp.tsx index 0031c127..ea6252e5 100644 --- a/apps/web/src/explorer/ExplorerApp.tsx +++ b/apps/web/src/explorer/ExplorerApp.tsx @@ -7,11 +7,27 @@ import { haversineKm } from '../globe/draw.js'; import { useSavedSearches } from '../state/savedSearches.js'; import { toast } from '../shell/toast.js'; import { Icon } from '../normal/Icon.js'; +import { apiFetch } from '../transport/http.js'; +import { useOntologySchema, type OntologySchema } from '../state/ontologySchema.js'; // Explorer app (design §6.1 / §8 "Object Explorer") — top-down analysis over the // live object store: type facets + keyword + rolling window, live counts, and a // tabular result set. Clicking a row selects the object (shared selection context) // and flies the camera to it. Backed by the real GET /api/search/objects. +// +// Two sources, and they are genuinely different questions: +// +// Live what is being emitted RIGHT NOW. A contact that stopped +// transponding is gone from it, because it is gone from the world. +// Ontology what was PROMOTED into the graph and kept. An object is here +// because somebody decided it mattered, and it stays after the +// feed forgets it. +// +// The ontology side is new: until GET /api/ontology/search existed the stored +// graph could only be reached by knowing an object's exact canonical id, so +// there was no way to browse what the platform had actually accumulated. Its +// facets and columns come from the declared schema (state/ontologySchema.ts), +// so the properties shown are the ones the kind is known to carry. const WINDOWS: Array<{ label: string; s: number | undefined }> = [ { label: 'All time', s: undefined }, @@ -27,7 +43,45 @@ function ageLabel(t: number): string { return `${Math.round(sec / 3600)}h`; } +/** One row of the stored graph, as the ontology routes return it. */ +interface OntObject { + id: string; + kind: string; + props: Record; +} + +type Source = 'live' | 'ontology'; + +/** The properties worth showing for a kind: the ones the schema declares, in + * declared order, falling back to whatever the object actually carries so an + * undeclared kind is still readable rather than blank. */ +function columnsFor( + schema: OntologySchema | null, + kind: string, + rows: OntObject[], +): string[] { + const declared = Object.keys(schema?.kinds?.[kind] ?? {}); + if (declared.length > 0) return declared.slice(0, 4); + const seen: string[] = []; + for (const r of rows) { + for (const k of Object.keys(r.props)) if (!seen.includes(k)) seen.push(k); + if (seen.length >= 4) break; + } + return seen.slice(0, 4); +} + +function cell(value: unknown): string { + // A lone em dash is the repo's "no value reported" (apps/web/CLAUDE.md §copy). + if (value === null || value === undefined || value === '') return '—'; + if (typeof value === 'object') return JSON.stringify(value).slice(0, 40); + return String(value); +} + export function ExplorerApp({ viewer }: { viewer: Cesium.Viewer | null }): JSX.Element { + const [source, setSource] = useState('live'); + const [ont, setOnt] = useState([]); + const [ontStatus, setOntStatus] = useState<'idle' | 'loading' | 'error'>('idle'); + const ontSchema = useOntologySchema(); const [type, setType] = useState('all'); const [q, setQ] = useState(''); const [winIdx, setWinIdx] = useState(0); @@ -41,7 +95,40 @@ export function ExplorerApp({ viewer }: { viewer: Cesium.Viewer | null }): JSX.E const saveSearch = useSavedSearches((s) => s.add); const abort = useRef(null); + // Ontology source: the stored graph. Debounced because every keystroke is an + // FTS query, and skipped entirely on an empty box — /api/ontology/search takes + // a query, and "everything ever promoted" is not a question this answers. useEffect(() => { + if (source !== 'ontology') return; + if (q.trim() === '') { + setOnt([]); + setOntStatus('idle'); + return; + } + const ac = new AbortController(); + setOntStatus('loading'); + const id = window.setTimeout(() => { + const params = new URLSearchParams({ q, limit: '200' }); + if (type !== 'all') params.append('kind', type); + apiFetch(`/api/ontology/search?${params.toString()}`, { signal: ac.signal }) + .then((r) => (r.ok ? (r.json() as Promise) : Promise.reject(new Error('http')))) + .then((rows) => { + setOnt(rows); + setOntStatus('idle'); + }) + .catch((e: unknown) => { + if ((e as { name?: string })?.name === 'AbortError') return; + setOntStatus('error'); + }); + }, 250); + return () => { + window.clearTimeout(id); + ac.abort(); + }; + }, [source, q, type]); + + useEffect(() => { + if (source !== 'live') return; abort.current?.abort(); const ac = new AbortController(); abort.current = ac; @@ -82,18 +169,38 @@ export function ExplorerApp({ viewer }: { viewer: Cesium.Viewer | null }): JSX.E }) .finally(() => setLoading(false)); return () => ac.abort(); - }, [type, q, winIdx, tick, geoScope]); + }, [source, type, q, winIdx, tick, geoScope]); - // Refresh on a slow tick so counts stay live without hammering. + // Refresh on a slow tick so counts stay live without hammering. The ontology + // is a durable store, not a feed, so it is not on this clock. useEffect(() => { + if (source !== 'live') return; const id = window.setInterval(() => setTick((n) => n + 1), 5000); return () => window.clearInterval(id); - }, []); + }, [source]); + + const ontByKind = useMemo(() => { + const counts: Record = {}; + for (const o of ont) counts[o.kind] = (counts[o.kind] ?? 0) + 1; + return counts; + }, [ont]); const typeChips = useMemo(() => { + if (source === 'ontology') { + const entries = Object.entries(ontByKind).sort((a, b) => b[1] - a[1]); + return [['all', ont.length] as [string, number], ...entries]; + } const entries = Object.entries(data.by_type).sort((a, b) => b[1] - a[1]); return [['all', data.count] as [string, number], ...entries]; - }, [data]); + }, [source, data, ont, ontByKind]); + + // In ontology mode the table's property columns follow the selected kind's + // declared schema, which is what makes this a typed explorer rather than a + // JSON dump. + const ontCols = useMemo( + () => columnsFor(ontSchema, type, ont), + [ontSchema, type, ont], + ); const exportCsv = (): void => { const rows = data.results; @@ -134,13 +241,41 @@ export function ExplorerApp({ viewer }: { viewer: Cesium.Viewer | null }): JSX.E
)}
+
+ {(['live', 'ontology'] as const).map((s) => ( + + ))} +
setQ(e.target.value)} - placeholder="Filter by callsign / name / id…" + placeholder={ + source === 'ontology' + ? 'Search the stored graph by any property…' + : 'Filter by callsign / name / id…' + } className="flex-1 min-w-0 bg-bg-0 border border-line rounded-sm px-2 py-1 text-[12px] text-txt-0 placeholder:text-txt-4 focus:border-accent-line outline-none" /> -
+
{WINDOWS.map((w, i) => (
- {data.results.length.toLocaleString()} shown · {data.count.toLocaleString()} match + {source === 'ontology' + ? `${ont.length.toLocaleString()} in the graph` + : `${data.results.length.toLocaleString()} shown · ${data.count.toLocaleString()} match`} -
+ {/* Both of these read the LIVE result set, which in ontology mode still + holds the last live fetch — Export CSV would download rows that are + not on screen and Save search would save a live-store subscription + from an ontology view. They belong to the live source only. */} +
{loading && updating…}