feat: Implement System Model, Mock Source and Ingestion

.github/workflows/verdict-backend.yaml

@@ -23,6 +23,8 @@ jobs:
 
       - uses: cachix/install-nix-action@v31
 
+      - uses: DeterminateSystems/magic-nix-cache-action@v7
+
       - uses: cachix/cachix-action@v16
         with:
           name: devenv
@@ -33,3 +35,11 @@ jobs:
       - name: Run tests
         run: devenv test
         working-directory: verdict-backend
+
+      - name: Publish test results
+        uses: dorny/test-reporter@v2
+        if: always()
+        with:
+          name: pytest
+          path: verdict-backend/test-results/*.xml
+          reporter: java-junit

.gitignore

@@ -24,6 +24,7 @@ ENV/
 .coverage
 htmlcov/
 .pytest_cache/
+test-results/
 
 # IDE
 .vscode/

devenv.lock

@@ -3,10 +3,11 @@
     "devenv": {
       "locked": {
         "dir": "src/modules",
-        "lastModified": 1771243888,
+        "lastModified": 1773077419,
+        "narHash": "sha256-Vr3Zrg4DEm4xy0StawiTEUs8TXruj1WdRZyDhlq1aPg=",
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "8e5434f65d6ca1db21564ae4295208a064d9de17",
+        "rev": "2105b1a0272e32b0d6a9e213b1c381ab4ae4a692",
         "type": "github"
       },
       "original": {
@@ -20,6 +21,7 @@
       "flake": false,
       "locked": {
         "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
         "owner": "NixOS",
         "repo": "flake-compat",
         "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
@@ -35,15 +37,14 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "gitignore": "gitignore",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
+        "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1770726378,
+        "lastModified": 1772893680,
+        "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "5eaaedde414f6eb1aea8b8525c466dc37bba95ae",
+        "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9",
         "type": "github"
       },
       "original": {
@@ -60,10 +61,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1762808025,
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
         "owner": "hercules-ci",
         "repo": "gitignore.nix",
-        "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
         "type": "github"
       },
       "original": {
@@ -73,31 +75,29 @@
       }
     },
     "nixpkgs": {
-      "inputs": {
-        "nixpkgs-src": "nixpkgs-src"
-      },
       "locked": {
-        "lastModified": 1770434727,
-        "owner": "cachix",
-        "repo": "devenv-nixpkgs",
-        "rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348",
+        "lastModified": 1770073757,
+        "narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "47472570b1e607482890801aeaf29bfb749884f6",
         "type": "github"
       },
       "original": {
-        "owner": "cachix",
-        "ref": "rolling",
-        "repo": "devenv-nixpkgs",
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1769922788,
-        "narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=",
+        "lastModified": 1772173633,
+        "narHash": "sha256-MOH58F4AIbCkh6qlQcwMycyk5SWvsqnS/TCfnqDlpj4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "207d15f1a6603226e1e223dc79ac29c7846da32e",
+        "rev": "c0f3d81a7ddbc2b1332be0d8481a672b4f6004d6",
         "type": "github"
       },
       "original": {
@@ -107,17 +107,33 @@
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "inputs": {
+        "nixpkgs-src": "nixpkgs-src"
+      },
+      "locked": {
+        "lastModified": 1772749504,
+        "narHash": "sha256-eqtQIz0alxkQPym+Zh/33gdDjkkch9o6eHnMPnXFXN0=",
+        "owner": "cachix",
+        "repo": "devenv-nixpkgs",
+        "rev": "08543693199362c1fddb8f52126030d0d374ba2e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "rolling",
+        "repo": "devenv-nixpkgs",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "devenv": "devenv",
         "git-hooks": "git-hooks",
-        "nixpkgs": "nixpkgs",
-        "pre-commit-hooks": [
-          "git-hooks"
-        ]
+        "nixpkgs": "nixpkgs_2"
       }
     }
   },
   "root": "root",
   "version": 7
-}
+}

devenv.yaml

@@ -1,2 +1,6 @@
+inputs:
+  git-hooks:
+    url: github:cachix/git-hooks.nix
+
 imports:
   - /shared

docs/patterns/error-handling.md

@@ -0,0 +1,25 @@
+# Error Handling
+
+The backend uses typed Result and Option types instead of exceptions for recoverable errors.
+
+## Result and Option
+
+A `Result[T, E]` is either `Ok(value)` or `Err(error)`, where the error type is explicit in the signature. An `Option[T]` is either `Some(value)` or `Nothing()`, representing the presence or absence of a value. These compose naturally: a database lookup returns `Result[Option[T], DBError]`, meaning the operation itself can fail (Err) or succeed with either a found record (Ok(Some)) or no record (Ok(Nothing)).
+
+## AppError Contract
+
+All error types inherit from `AppError`, an abstract base class that enforces a two-level message contract. Each error exposes a `message` property (safe to return to clients) and a `detail` property (for internal logging only, may contain URLs, SQL statements, or driver messages). `__str__` returns `message`, so errors are safe by default anywhere they are converted to strings. Error types store their raw diagnostic data in fields and compute both properties from them.
+
+Error types are domain-specific dataclasses defined in `app/errors.py`. They form unions that describe what can go wrong in each context: `IngestionError` covers fetch failures, schema validation errors, and database errors. Functions declare which error union they can produce, and callers handle each variant explicitly.
+
+## Route Helpers
+
+Routes bridge typed errors to HTTP responses via two helpers: `unwrap_or_raise` converts a `Result[T, E]` into either the unwrapped value or an HTTP error, while `unwrap_optional_or_raise` handles `Result[Option[T], E]` with an additional Nothing-to-404 mapping. Both log the error detail internally and return only the safe message to clients.
+
+## Defence in Depth
+
+The SQLAlchemy engine is configured with `hide_parameters=True`, which redacts SQL parameter values from exception messages and query logging at the driver level. This is a defence-in-depth measure alongside the `AppError` message/detail separation.
+
+## Exceptions vs Results
+
+Exceptions are reserved for programming errors (bugs) that should propagate and crash. Operational errors (network failures, missing records, schema mismatches) flow through Result types so they can be handled, composed, and tested without try/except.

docs/patterns/ingestion.md

@@ -0,0 +1,27 @@
+# Ingestion
+
+## Single-Source Pipeline
+
+External data is ingested through a three-phase pipeline that separates fetching from persistence.
+
+**Phase 1 — Fetch Index.** The service calls the external source's list endpoint, which returns a slim representation of each item (typically just an ID and a name). This is validated against a Pydantic model specific to the index response shape.
+
+**Phase 2 — Fetch Detail.** For each item in the index, the service calls the detail endpoint to retrieve the full representation. This is validated against a separate Pydantic model for the detail response shape.
+
+**Phase 3 — Convert and Upsert.** A pure function maps each external detail to an `XCreate` model instance. The orchestrator then upserts each converted record into the database, keyed by gold source ID (the record's unique identifier in the external system — see [Gold Source Identity](models.md#gold-source-identity)), and returns `XPublic` instances via `model_validate` after flush.
+
+The separation of index and detail schemas reflects the reality that list and detail endpoints often return different shapes. Keeping the conversion as a pure function (no IO, no session) makes it independently testable. The orchestrator does not commit — the caller (typically the route handler) owns the session lifecycle, which preserves all-or-nothing semantics: if any phase fails, nothing is persisted.
+
+## Cross-Source Orchestration
+
+When multiple sources need to be ingested atomically, a global ingestion endpoint (`POST /ingest`) runs each source's pipeline in sequence within a single session. It commits only if all sources succeed. If any source fails, the session is rolled back and nothing is persisted.
+
+Sources that reference records from other sources (e.g., systems referencing assets) resolve those references by gold source ID during their pipeline. If a referenced record cannot be found, the entire source's ingestion is rejected.
+
+## Link Resolution and Syncing
+
+For N:M relationships that cross source boundaries, the ingestion pipeline resolves external IDs to internal IDs using `get_by_gold_source`. After upserting the parent record, a diff-based sync step compares the desired set of linked IDs against the current set in the database, then issues INSERT and DELETE statements against the join table to reconcile the difference. This ensures re-ingestion is idempotent — links are added or removed to match the source of truth without duplicating or orphaning entries.
+
+## External Schemas
+
+Pydantic models for external API responses live under `app/schemas/external/`, separate from internal response schemas. Each external source has its own module with an index model (slim, for list responses) and a detail model (full, for single-item responses). These models describe the external contract, not the internal representation — the conversion function bridges the two.

docs/patterns/mock-services.md

@@ -0,0 +1,17 @@
+# Mock Services
+
+External APIs are simulated by lightweight FastAPI applications under `mock-services/`. Each mock service loads fixture data from YAML files and serves it through endpoints that mirror the real external API structure (index and detail).
+
+## Factory
+
+Mock services are built using `create_mock_app` from `mock-services/mock_helpers.py`. The factory takes a data directory, a Pydantic model class, and the item's ID field name, then returns a FastAPI app with index and detail endpoints. This avoids duplicating the same boilerplate across mock services — each mock module only needs to define its models and point to its YAML fixture file.
+
+## YAML Fixtures
+
+Fixture data lives in `<mock-service>/data/*.yaml`. Each file contains a list of records that the mock serves. The `load_yaml` helper resolves paths and enforces that they fall within the `mock-services/` base directory to prevent path traversal.
+
+## Usage
+
+Mock services are used both in local development (started via devenv process manager) and in tests. E2E tests start mock services as background processes and configure the backend to point at them. Unit and integration tests use `respx` to mock HTTP calls instead, avoiding the need for running mock services.
+
+Mock services are intentionally simple and untested — their purpose is to provide deterministic, schema-valid responses for development and testing.