diff --git a/.github/workflows/verdict-backend.yaml b/.github/workflows/verdict-backend.yaml index 2592b42..066ccef 100644 --- a/.github/workflows/verdict-backend.yaml +++ b/.github/workflows/verdict-backend.yaml @@ -23,6 +23,8 @@ jobs: - uses: cachix/install-nix-action@v31 + - uses: DeterminateSystems/magic-nix-cache-action@v7 + - uses: cachix/cachix-action@v16 with: name: devenv @@ -33,3 +35,11 @@ jobs: - name: Run tests run: devenv test working-directory: verdict-backend + + - name: Publish test results + uses: dorny/test-reporter@v2 + if: always() + with: + name: pytest + path: verdict-backend/test-results/*.xml + reporter: java-junit diff --git a/.gitignore b/.gitignore index 8a274e4..e94ab2f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ ENV/ .coverage htmlcov/ .pytest_cache/ +test-results/ # IDE .vscode/ diff --git a/devenv.lock b/devenv.lock index f7f1e9c..307ef30 100644 --- a/devenv.lock +++ b/devenv.lock @@ -3,10 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1771243888, + "lastModified": 1773077419, + "narHash": "sha256-Vr3Zrg4DEm4xy0StawiTEUs8TXruj1WdRZyDhlq1aPg=", "owner": "cachix", "repo": "devenv", - "rev": "8e5434f65d6ca1db21564ae4295208a064d9de17", + "rev": "2105b1a0272e32b0d6a9e213b1c381ab4ae4a692", "type": "github" }, "original": { @@ -20,6 +21,7 @@ "flake": false, "locked": { "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", "owner": "NixOS", "repo": "flake-compat", "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", @@ -35,15 +37,14 @@ "inputs": { "flake-compat": "flake-compat", "gitignore": "gitignore", - "nixpkgs": [ - "nixpkgs" - ] + "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1770726378, + "lastModified": 1772893680, + "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "5eaaedde414f6eb1aea8b8525c466dc37bba95ae", + "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9", "type": "github" }, "original": { @@ -60,10 +61,11 @@ ] }, "locked": { - "lastModified": 1762808025, + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", "owner": "hercules-ci", "repo": "gitignore.nix", - "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", "type": "github" }, "original": { @@ -73,31 +75,29 @@ } }, "nixpkgs": { - "inputs": { - "nixpkgs-src": "nixpkgs-src" - }, "locked": { - "lastModified": 1770434727, - "owner": "cachix", - "repo": "devenv-nixpkgs", - "rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348", + "lastModified": 1770073757, + "narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "47472570b1e607482890801aeaf29bfb749884f6", "type": "github" }, "original": { - "owner": "cachix", - "ref": "rolling", - "repo": "devenv-nixpkgs", + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", "type": "github" } }, "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1769922788, - "narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=", + "lastModified": 1772173633, + "narHash": "sha256-MOH58F4AIbCkh6qlQcwMycyk5SWvsqnS/TCfnqDlpj4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "207d15f1a6603226e1e223dc79ac29c7846da32e", + "rev": "c0f3d81a7ddbc2b1332be0d8481a672b4f6004d6", "type": "github" }, "original": { @@ -107,17 +107,33 @@ "type": "github" } }, + "nixpkgs_2": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1772749504, + "narHash": "sha256-eqtQIz0alxkQPym+Zh/33gdDjkkch9o6eHnMPnXFXN0=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "08543693199362c1fddb8f52126030d0d374ba2e", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "devenv": "devenv", "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs", - "pre-commit-hooks": [ - "git-hooks" - ] + "nixpkgs": "nixpkgs_2" } } }, "root": "root", "version": 7 -} +} \ No newline at end of file diff --git a/devenv.yaml b/devenv.yaml index cc42a76..bbf2ce9 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -1,2 +1,6 @@ +inputs: + git-hooks: + url: github:cachix/git-hooks.nix + imports: - /shared diff --git a/docs/patterns/backend.md b/docs/patterns/backend.md deleted file mode 100644 index e102d1a..0000000 --- a/docs/patterns/backend.md +++ /dev/null @@ -1,47 +0,0 @@ -# Backend Patterns - -## Error Handling - -The backend uses typed Result and Option types instead of exceptions for recoverable errors. - -A `Result[T, E]` is either `Ok(value)` or `Err(error)`, where the error type is explicit in the signature. An `Option[T]` is either `Some(value)` or `Nothing()`, representing the presence or absence of a value. These compose naturally: a database lookup returns `Result[Option[T], DBError]`, meaning the operation itself can fail (Err) or succeed with either a found record (Ok(Some)) or no record (Ok(Nothing)). - -All error types inherit from `AppError`, an abstract base class that enforces a two-level message contract. Each error exposes a `message` property (safe to return to clients) and a `detail` property (for internal logging only, may contain URLs, SQL statements, or driver messages). `__str__` returns `message`, so errors are safe by default anywhere they are converted to strings. Error types store their raw diagnostic data in fields and compute both properties from them. - -Error types are domain-specific dataclasses defined in `app/errors.py`. They form unions that describe what can go wrong in each context: `IngestionError` covers fetch failures, schema validation errors, and database errors. Functions declare which error union they can produce, and callers handle each variant explicitly. - -Routes bridge typed errors to HTTP responses via two helpers: `unwrap_or_raise` converts a `Result[T, E]` into either the unwrapped value or an HTTP error, while `unwrap_optional_or_raise` handles `Result[Option[T], E]` with an additional Nothing-to-404 mapping. Both log the error detail internally and return only the safe message to clients. - -The SQLAlchemy engine is configured with `hide_parameters=True`, which redacts SQL parameter values from exception messages and query logging at the driver level. This is a defence-in-depth measure alongside the `AppError` message/detail separation. - -Exceptions are reserved for programming errors (bugs) that should propagate and crash. Operational errors (network failures, missing records, schema mismatches) flow through Result types so they can be handled, composed, and tested without try/except. - -## Ingestion Pipeline - -External data is ingested through a three-phase pipeline that separates fetching from persistence. - -**Phase 1 — Fetch Index.** The service calls the external source's list endpoint, which returns a slim representation of each item (typically just an ID and a name). This is validated against a Pydantic model specific to the index response shape. - -**Phase 2 — Fetch Detail.** For each item in the index, the service calls the detail endpoint to retrieve the full representation. This is validated against a separate Pydantic model for the detail response shape. - -**Phase 3 — Convert and Upsert.** A pure function maps each external detail to an internal model instance. The orchestrator then upserts each converted record into the database, keyed by gold source ID. - -The separation of index and detail schemas reflects the reality that list and detail endpoints often return different shapes. Keeping the conversion as a pure function (no IO, no session) makes it independently testable. The orchestrator does not commit — the caller (typically the route handler) owns the session lifecycle, which preserves all-or-nothing semantics: if any phase fails, nothing is persisted. - -## External Schemas - -Pydantic models for external API responses live under `app/schemas/external/`, separate from internal response schemas. Each external source has its own module with an index model (slim, for list responses) and a detail model (full, for single-item responses). These models describe the external contract, not the internal representation — the conversion function bridges the two. - -## Gold Source Identity - -Models that are synced from external systems inherit from `GoldSourceMixin`, which provides `gold_source_id` and `gold_source_type` fields with a composite unique constraint. This allows any synced record to be looked up by its external identity. Each model adds a composite index on these fields for query performance. The mixin provides a `get_by_gold_source` query method that returns a typed `Result[Option[T], DBError]`, consistent with the error handling pattern. - -## Mock Services - -External APIs are simulated by lightweight FastAPI applications under `mock-services/`. Each mock service loads fixture data from YAML files and serves it through endpoints that mirror the real external API structure (index and detail). Mock services are used both in local development (started via devenv) and in CI (started as background processes during test runs). They are intentionally simple and untested — their purpose is to provide deterministic, schema-valid responses for development and testing. - -## Routes and Responses - -Route handlers are thin. They call into service functions or query methods, receive typed Results, and convert them to HTTP responses. List endpoints support offset/limit pagination. Lookup endpoints return 404 when no record matches. Ingestion endpoints trigger the pipeline, commit on success, and return the count of ingested records or a 502 on failure. - -Internal response schemas (under `app/schemas/`) define the shape of API responses. These are separate from both the database models and the external schemas — each layer has its own representation. diff --git a/docs/patterns/error-handling.md b/docs/patterns/error-handling.md new file mode 100644 index 0000000..f344e43 --- /dev/null +++ b/docs/patterns/error-handling.md @@ -0,0 +1,25 @@ +# Error Handling + +The backend uses typed Result and Option types instead of exceptions for recoverable errors. + +## Result and Option + +A `Result[T, E]` is either `Ok(value)` or `Err(error)`, where the error type is explicit in the signature. An `Option[T]` is either `Some(value)` or `Nothing()`, representing the presence or absence of a value. These compose naturally: a database lookup returns `Result[Option[T], DBError]`, meaning the operation itself can fail (Err) or succeed with either a found record (Ok(Some)) or no record (Ok(Nothing)). + +## AppError Contract + +All error types inherit from `AppError`, an abstract base class that enforces a two-level message contract. Each error exposes a `message` property (safe to return to clients) and a `detail` property (for internal logging only, may contain URLs, SQL statements, or driver messages). `__str__` returns `message`, so errors are safe by default anywhere they are converted to strings. Error types store their raw diagnostic data in fields and compute both properties from them. + +Error types are domain-specific dataclasses defined in `app/errors.py`. They form unions that describe what can go wrong in each context: `IngestionError` covers fetch failures, schema validation errors, and database errors. Functions declare which error union they can produce, and callers handle each variant explicitly. + +## Route Helpers + +Routes bridge typed errors to HTTP responses via two helpers: `unwrap_or_raise` converts a `Result[T, E]` into either the unwrapped value or an HTTP error, while `unwrap_optional_or_raise` handles `Result[Option[T], E]` with an additional Nothing-to-404 mapping. Both log the error detail internally and return only the safe message to clients. + +## Defence in Depth + +The SQLAlchemy engine is configured with `hide_parameters=True`, which redacts SQL parameter values from exception messages and query logging at the driver level. This is a defence-in-depth measure alongside the `AppError` message/detail separation. + +## Exceptions vs Results + +Exceptions are reserved for programming errors (bugs) that should propagate and crash. Operational errors (network failures, missing records, schema mismatches) flow through Result types so they can be handled, composed, and tested without try/except. diff --git a/docs/patterns/ingestion.md b/docs/patterns/ingestion.md new file mode 100644 index 0000000..1111f06 --- /dev/null +++ b/docs/patterns/ingestion.md @@ -0,0 +1,27 @@ +# Ingestion + +## Single-Source Pipeline + +External data is ingested through a three-phase pipeline that separates fetching from persistence. + +**Phase 1 — Fetch Index.** The service calls the external source's list endpoint, which returns a slim representation of each item (typically just an ID and a name). This is validated against a Pydantic model specific to the index response shape. + +**Phase 2 — Fetch Detail.** For each item in the index, the service calls the detail endpoint to retrieve the full representation. This is validated against a separate Pydantic model for the detail response shape. + +**Phase 3 — Convert and Upsert.** A pure function maps each external detail to an `XCreate` model instance. The orchestrator then upserts each converted record into the database, keyed by gold source ID (the record's unique identifier in the external system — see [Gold Source Identity](models.md#gold-source-identity)), and returns `XPublic` instances via `model_validate` after flush. + +The separation of index and detail schemas reflects the reality that list and detail endpoints often return different shapes. Keeping the conversion as a pure function (no IO, no session) makes it independently testable. The orchestrator does not commit — the caller (typically the route handler) owns the session lifecycle, which preserves all-or-nothing semantics: if any phase fails, nothing is persisted. + +## Cross-Source Orchestration + +When multiple sources need to be ingested atomically, a global ingestion endpoint (`POST /ingest`) runs each source's pipeline in sequence within a single session. It commits only if all sources succeed. If any source fails, the session is rolled back and nothing is persisted. + +Sources that reference records from other sources (e.g., systems referencing assets) resolve those references by gold source ID during their pipeline. If a referenced record cannot be found, the entire source's ingestion is rejected. + +## Link Resolution and Syncing + +For N:M relationships that cross source boundaries, the ingestion pipeline resolves external IDs to internal IDs using `get_by_gold_source`. After upserting the parent record, a diff-based sync step compares the desired set of linked IDs against the current set in the database, then issues INSERT and DELETE statements against the join table to reconcile the difference. This ensures re-ingestion is idempotent — links are added or removed to match the source of truth without duplicating or orphaning entries. + +## External Schemas + +Pydantic models for external API responses live under `app/schemas/external/`, separate from internal response schemas. Each external source has its own module with an index model (slim, for list responses) and a detail model (full, for single-item responses). These models describe the external contract, not the internal representation — the conversion function bridges the two. diff --git a/docs/patterns/mock-services.md b/docs/patterns/mock-services.md new file mode 100644 index 0000000..73afde9 --- /dev/null +++ b/docs/patterns/mock-services.md @@ -0,0 +1,17 @@ +# Mock Services + +External APIs are simulated by lightweight FastAPI applications under `mock-services/`. Each mock service loads fixture data from YAML files and serves it through endpoints that mirror the real external API structure (index and detail). + +## Factory + +Mock services are built using `create_mock_app` from `mock-services/mock_helpers.py`. The factory takes a data directory, a Pydantic model class, and the item's ID field name, then returns a FastAPI app with index and detail endpoints. This avoids duplicating the same boilerplate across mock services — each mock module only needs to define its models and point to its YAML fixture file. + +## YAML Fixtures + +Fixture data lives in `/data/*.yaml`. Each file contains a list of records that the mock serves. The `load_yaml` helper resolves paths and enforces that they fall within the `mock-services/` base directory to prevent path traversal. + +## Usage + +Mock services are used both in local development (started via devenv process manager) and in tests. E2E tests start mock services as background processes and configure the backend to point at them. Unit and integration tests use `respx` to mock HTTP calls instead, avoiding the need for running mock services. + +Mock services are intentionally simple and untested — their purpose is to provide deterministic, schema-valid responses for development and testing. diff --git a/docs/patterns/models.md b/docs/patterns/models.md new file mode 100644 index 0000000..92386f2 --- /dev/null +++ b/docs/patterns/models.md @@ -0,0 +1,45 @@ +# Models + +## Multi-Model Hierarchy + +Each domain entity uses a family of SQLModel classes that separate concerns across layers. + +`XBase` carries the domain-specific fields shared by all variants (e.g., `name`, `description` for assets). It inherits from domain mixins (`GoldSourceMixin`, `TagsMixin`) and `SQLModel`, but not from `BaseModel` — it has no id, no timestamps, and no table mapping. + +`X` (the table model) inherits from both `XBase` and `BaseModel`, with `table=True`. It gains `id`, `created_at`, `updated_at` from `BaseModel`, plus any ORM relationships. This is the only class that maps to a database table. + +`XCreate` inherits from `XBase` only. It carries the fields needed to create a new record — domain fields plus gold source identity — but no id or timestamps, since those are generated by the database. + +`XPublic` inherits from both `XBase` and `PublicModel`. It provides the read representation with narrowed types: `id: int` (not `int | None`), `created_at: datetime` and `updated_at: datetime` (not `datetime | None`). These are guaranteed to be populated after a database flush. `XPublic` also serves as the API response type — there is no separate response schema. Where a model has relationships, `XPublic` includes derived fields (e.g., `asset_ids: list[int]`) that are populated during conversion from the ORM model. + +## BaseModel and PublicModel + +`BaseModel` (`app/models/base_model.py`) is the base for all table models. It provides: + +- `id: int | None` — auto-increment primary key, `None` before flush +- `created_at: datetime | None` — server-default `now()`, `None` before flush +- `updated_at: datetime | None` — server-default `now()`, updated on every write via `clock_timestamp()` + +`PublicModel` (`app/models/base_model.py`) is the base for all public/read models. It narrows the same three fields to their non-optional forms: `id: int`, `created_at: datetime`, `updated_at: datetime`. Because `PublicModel` does not inherit from `BaseModel`, there is no override conflict — the fields are declared fresh. + +## Gold Source Identity + +Models that are synced from external systems inherit from `GoldSourceMixin`, which provides `gold_source_id` and `gold_source_type` fields with a composite unique constraint. This allows any synced record to be looked up by its external identity. Each model adds a composite index on these fields for query performance. The mixin provides a `get_by_gold_source` query method that returns a typed `Result[Option[T], DBError]`, consistent with the error handling pattern. + +## Query Functions and Type Narrowing + +The query functions `get_by_id`, `get_paginated`, and `get_by_gold_source` accept an optional `public_class` keyword parameter. When provided, the function calls `model_validate` on the result internally and returns the narrowed public type (e.g., `Result[Option[AssetPublic], DBError]`). This avoids the need for `assert` guards or manual `model_validate` calls at every call site. The parameter is typed via `@overload` so callers get precise return types. + +## N:M Relationships + +Many-to-many relationships use a plain SQLAlchemy `sa.Table` for the join table, combined with an ORM `Relationship` on the table model. Join tables are defined in the module of the model that owns the relationship (e.g., `asset_system` lives in `system.py` because systems declare which assets they contain). + +Relationships use `lazy="selectin"` for eager loading — related records are fetched in a single additional SELECT rather than lazily on attribute access. Where the relationship is informational and should not cascade writes, `viewonly=True` is set (e.g., `Asset.systems` is viewonly; `System.assets` is not). + +In `XPublic`, relationships are represented as flat ID lists (e.g., `asset_ids: list[int]`) rather than nested objects, keeping responses shallow and predictable. A `_to_public` helper in the route module converts the ORM model to `XPublic`, extracting IDs from the loaded relationship. + +## SQLModel and TYPE_CHECKING + +`from __future__ import annotations` is incompatible with SQLModel's `Relationship` — it turns all annotations into strings, and SQLAlchemy misinterprets generic annotations like `list[System]` as class names. Instead, forward references use explicit string annotations: `list["System"]`. + +For the type checker to resolve these string annotations, the referenced model must be imported under `TYPE_CHECKING`. This import is erased at runtime (avoiding circular imports between model modules) but visible to mypy and pyright. diff --git a/docs/patterns/routes.md b/docs/patterns/routes.md new file mode 100644 index 0000000..8f34dd1 --- /dev/null +++ b/docs/patterns/routes.md @@ -0,0 +1,17 @@ +# Routes and Responses + +## Route Handlers + +Route handlers are thin. They call into service functions or query methods, receive typed Results, and convert them to HTTP responses using the `unwrap_or_raise` / `unwrap_optional_or_raise` helpers from the error handling pattern. + +List endpoints support offset/limit pagination. Lookup endpoints return 404 when no record matches. Ingestion endpoints trigger the pipeline, commit on success, and return the count of ingested records. On failure they return 502 Bad Gateway, since the ingestion endpoint acts as a gateway to external sources and failures originate upstream (fetch errors, invalid responses). + +## Response Types + +`XPublic` models (defined in `app/models/`) serve directly as API response types. There is no separate response schema layer — `XPublic` is used as the `response_model` in route decorators and as the return type of route handlers. + +For simple models (no relationships), routes pass `public_class=XPublic` to query functions (`get_by_id`, `get_paginated`, `get_by_gold_source`), which return the narrowed type directly. + +For models with relationships that need derived fields (e.g., `asset_ids`), routes query the ORM model and convert via a `_to_public` helper that populates the derived fields from the loaded relationship. + +List endpoints use lightweight wrapper schemas (e.g., `AssetListResponse`) under `app/schemas/` that pair a list of `XPublic` items with a `total` count. These are the only response schemas — individual item responses use `XPublic` directly. diff --git a/mock-services/asset_inventory/app.py b/mock-services/asset_inventory/app.py index d4c299f..cfb79d1 100644 --- a/mock-services/asset_inventory/app.py +++ b/mock-services/asset_inventory/app.py @@ -1,34 +1,12 @@ -# Mock server for local development and testing -- not for production use, -# and intentionally untested. - from pathlib import Path -import yaml -from fastapi import FastAPI, HTTPException - from asset_inventory.models import AssetIndexItem, AssetItem - -_data_dir = Path(__file__).parent / "data" - -with (_data_dir / "assets.yaml").open() as f: - _raw = yaml.safe_load(f) - -_items: list[AssetItem] = [AssetItem.model_validate(item) for item in _raw] -_index: dict[str, AssetItem] = {item.id: item for item in _items} - -app = FastAPI(title="Asset Inventory Mock") - - -@app.get("/assets", response_model=list[AssetIndexItem]) -def list_assets(offset: int = 0, limit: int = 50) -> list[AssetIndexItem]: - return [ - AssetIndexItem(id=item.id, name=item.name) - for item in _items[offset : offset + limit] - ] - - -@app.get("/assets/{asset_id}", response_model=AssetItem) -def get_asset(asset_id: str) -> AssetItem: - if asset_id not in _index: - raise HTTPException(status_code=404, detail=f"Asset {asset_id} not found") - return _index[asset_id] +from mock_helpers import create_mock_app + +app, _ = create_mock_app( + title="Asset Inventory Mock", + data_path=Path(__file__).parent / "data" / "assets.yaml", + full_model=AssetItem, + index_model=AssetIndexItem, + resource_name="assets", +) diff --git a/mock-services/cmdb/__init__.py b/mock-services/cmdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mock-services/cmdb/app.py b/mock-services/cmdb/app.py new file mode 100644 index 0000000..87bbd25 --- /dev/null +++ b/mock-services/cmdb/app.py @@ -0,0 +1,12 @@ +from pathlib import Path + +from cmdb.models import SystemIndexItem, SystemItem +from mock_helpers import create_mock_app + +app, _ = create_mock_app( + title="CMDB Mock", + data_path=Path(__file__).parent / "data" / "systems.yaml", + full_model=SystemItem, + index_model=SystemIndexItem, + resource_name="systems", +) diff --git a/mock-services/cmdb/data/systems.yaml b/mock-services/cmdb/data/systems.yaml new file mode 100644 index 0000000..1bf29d9 --- /dev/null +++ b/mock-services/cmdb/data/systems.yaml @@ -0,0 +1,41 @@ +- id: "SYS-001" + primary_fqdn: "banking-web-01.prod.example.com" + asset_gold_source_ids: + - "SVC-001" + tags: + - "env:production" + - "tier:frontend" + +- id: "SYS-002" + primary_fqdn: "shared-db-01.prod.example.com" + asset_gold_source_ids: + - "SVC-001" + - "SVC-002" + tags: + - "env:production" + - "tier:data" + - "role:database" + +- id: "SYS-003" + primary_fqdn: "monitoring-01.infra.example.com" + asset_gold_source_ids: [] + tags: + - "env:production" + - "tier:infrastructure" + +- id: "SYS-004" + primary_fqdn: "banking-api-01.prod.example.com" + asset_gold_source_ids: + - "SVC-001" + tags: + - "env:production" + - "tier:backend" + +- id: "SYS-005" + primary_fqdn: "payment-gw-01.prod.example.com" + asset_gold_source_ids: + - "SVC-003" + tags: + - "env:production" + - "tier:backend" + - "compliance:pci-dss" diff --git a/mock-services/cmdb/models.py b/mock-services/cmdb/models.py new file mode 100644 index 0000000..e0f5a45 --- /dev/null +++ b/mock-services/cmdb/models.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel + + +class SystemIndexItem(BaseModel): + id: str + primary_fqdn: str + + +class SystemItem(BaseModel): + id: str + primary_fqdn: str + asset_gold_source_ids: list[str] + tags: list[str] diff --git a/mock-services/devenv.lock b/mock-services/devenv.lock index 92200d2..560acb6 100644 --- a/mock-services/devenv.lock +++ b/mock-services/devenv.lock @@ -3,10 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1772049386, + "lastModified": 1773077419, + "narHash": "sha256-Vr3Zrg4DEm4xy0StawiTEUs8TXruj1WdRZyDhlq1aPg=", "owner": "cachix", "repo": "devenv", - "rev": "e9f3b7808443cd3d5b492863bc9c839f357e73da", + "rev": "2105b1a0272e32b0d6a9e213b1c381ab4ae4a692", "type": "github" }, "original": { @@ -19,85 +20,67 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1767039857, - "owner": "NixOS", + "lastModified": 1747046372, + "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", + "owner": "edolstra", "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", "type": "github" }, "original": { - "owner": "NixOS", + "owner": "edolstra", "repo": "flake-compat", "type": "github" } }, - "git-hooks": { + "nixpkgs": { "inputs": { - "flake-compat": "flake-compat", - "gitignore": "gitignore", - "nixpkgs": [ - "nixpkgs" - ] + "nixpkgs-src": "nixpkgs-src" }, "locked": { - "lastModified": 1772024342, + "lastModified": 1772749504, + "narHash": "sha256-eqtQIz0alxkQPym+Zh/33gdDjkkch9o6eHnMPnXFXN0=", "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "6e34e97ed9788b17796ee43ccdbaf871a5c2b476", + "repo": "devenv-nixpkgs", + "rev": "08543693199362c1fddb8f52126030d0d374ba2e", "type": "github" }, "original": { "owner": "cachix", - "repo": "git-hooks.nix", + "ref": "rolling", + "repo": "devenv-nixpkgs", "type": "github" } }, - "gitignore": { + "nixpkgs-python": { "inputs": { + "flake-compat": "flake-compat", "nixpkgs": [ - "git-hooks", "nixpkgs" ] }, "locked": { - "lastModified": 1762808025, - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "nixpkgs": { - "inputs": { - "nixpkgs-src": "nixpkgs-src" - }, - "locked": { - "lastModified": 1770434727, + "lastModified": 1772559541, + "narHash": "sha256-1cWCYOnOz3K7FchI9IbIbu1qhi68mSNZSuYlE5jEv1I=", "owner": "cachix", - "repo": "devenv-nixpkgs", - "rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348", + "repo": "nixpkgs-python", + "rev": "188ffc273f679a2d95a1c8aba0be083545417bdd", "type": "github" }, "original": { "owner": "cachix", - "ref": "rolling", - "repo": "devenv-nixpkgs", + "repo": "nixpkgs-python", "type": "github" } }, "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1769922788, - "narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=", + "lastModified": 1772173633, + "narHash": "sha256-MOH58F4AIbCkh6qlQcwMycyk5SWvsqnS/TCfnqDlpj4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "207d15f1a6603226e1e223dc79ac29c7846da32e", + "rev": "c0f3d81a7ddbc2b1332be0d8481a672b4f6004d6", "type": "github" }, "original": { @@ -110,14 +93,11 @@ "root": { "inputs": { "devenv": "devenv", - "git-hooks": "git-hooks", "nixpkgs": "nixpkgs", - "pre-commit-hooks": [ - "git-hooks" - ] + "nixpkgs-python": "nixpkgs-python" } } }, "root": "root", "version": 7 -} +} \ No newline at end of file diff --git a/mock-services/devenv.nix b/mock-services/devenv.nix index 048a9f9..3b15667 100644 --- a/mock-services/devenv.nix +++ b/mock-services/devenv.nix @@ -32,4 +32,19 @@ in }; }; }; + + processes.cmdb-mock = { + exec = "cd $MOCK_SERVICES_DIR && uv run uvicorn cmdb.app:app --host 0.0.0.0 --port 4011"; + process-compose = { + readiness_probe = { + http_get = { + host = "localhost"; + port = 4011; + path = "/systems"; + }; + initial_delay_seconds = 2; + period_seconds = 2; + }; + }; + }; } diff --git a/mock-services/justfile b/mock-services/justfile index 9b8437c..35302d0 100644 --- a/mock-services/justfile +++ b/mock-services/justfile @@ -5,6 +5,10 @@ set dotenv-load default: @just --list -# Start all mock services -mock: +# Start the asset inventory mock +mock-asset-inventory: uv run uvicorn asset_inventory.app:app --host 0.0.0.0 --port 4010 + +# Start the CMDB mock +mock-cmdb: + uv run uvicorn cmdb.app:app --host 0.0.0.0 --port 4011 diff --git a/mock-services/mock_helpers.py b/mock-services/mock_helpers.py new file mode 100644 index 0000000..9ed57a3 --- /dev/null +++ b/mock-services/mock_helpers.py @@ -0,0 +1,54 @@ +from pathlib import Path +from typing import Any + +import yaml +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel + +_BASE_DIR = Path(__file__).resolve().parent + + +def load_yaml(path: Path) -> list[dict[str, Any]]: + resolved = path.resolve() + if not resolved.is_relative_to(_BASE_DIR): + raise ValueError(f"path {resolved} is outside base directory {_BASE_DIR}") + with resolved.open() as f: + data = yaml.safe_load(f) + if data is None: + return [] + if not isinstance(data, list): + raise ValueError(f"expected a list in {resolved}, got {type(data).__name__}") + return data + + +def create_mock_app[TFull: BaseModel, TIndex: BaseModel]( + *, + title: str, + data_path: Path, + full_model: type[TFull], + index_model: type[TIndex], + id_field: str = "id", + resource_name: str, +) -> tuple[FastAPI, list[TFull]]: + raw = load_yaml(data_path) + items: list[TFull] = [full_model.model_validate(item) for item in raw] + index: dict[str, TFull] = {str(getattr(item, id_field)): item for item in items} + + app = FastAPI(title=title) + + @app.get(f"/{resource_name}", response_model=list[index_model]) + def list_items(offset: int = 0, limit: int = 50) -> list[index_model]: + return [ + index_model.model_validate(item, from_attributes=True) + for item in items[offset : offset + limit] + ] + + @app.get(f"/{resource_name}/{{item_id}}", response_model=full_model) + def get_item(item_id: str) -> TFull: + if item_id not in index: + raise HTTPException( + status_code=404, detail=f"{resource_name} {item_id} not found" + ) + return index[item_id] + + return app, items diff --git a/verdict-backend/alembic/versions/3bfa8f122226_create_system_and_asset_system_tables.py b/verdict-backend/alembic/versions/3bfa8f122226_create_system_and_asset_system_tables.py new file mode 100644 index 0000000..271006d --- /dev/null +++ b/verdict-backend/alembic/versions/3bfa8f122226_create_system_and_asset_system_tables.py @@ -0,0 +1,59 @@ +"""create system and asset_system tables + +Revision ID: 3bfa8f122226 +Revises: 6b960c32fa67 +Create Date: 2026-03-03 22:34:22.098102 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "3bfa8f122226" +down_revision: str | Sequence[str] | None = "6b960c32fa67" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "system", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tags", sa.JSON(), server_default="[]", nullable=False), + sa.Column("gold_source_id", sa.String(), nullable=False), + sa.Column("gold_source_type", sa.String(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("primary_fqdn", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("gold_source_type", "gold_source_id"), + ) + op.create_table( + "asset_system", + sa.Column("asset_id", sa.Integer(), nullable=False), + sa.Column("system_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["asset_id"], ["asset.id"]), + sa.ForeignKeyConstraint(["system_id"], ["system.id"]), + sa.PrimaryKeyConstraint("asset_id", "system_id"), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_table("asset_system") + op.drop_table("system") diff --git a/verdict-backend/app/config.py b/verdict-backend/app/config.py index 103a4eb..b36c2df 100644 --- a/verdict-backend/app/config.py +++ b/verdict-backend/app/config.py @@ -20,6 +20,7 @@ class Settings(BaseSettings): database_password: SecretStr = Field(default=...) asset_inventory_url: str = Field(default=...) + cmdb_url: str = Field(default=...) @property def database_url(self) -> str: diff --git a/verdict-backend/app/main.py b/verdict-backend/app/main.py index 24c869a..91d2cec 100644 --- a/verdict-backend/app/main.py +++ b/verdict-backend/app/main.py @@ -1,9 +1,13 @@ from fastapi import FastAPI from app.routes.assets import router as assets_router +from app.routes.ingestion import router as ingestion_router +from app.routes.systems import router as systems_router app = FastAPI() app.include_router(assets_router) +app.include_router(systems_router) +app.include_router(ingestion_router) @app.get("/") diff --git a/verdict-backend/app/models/__init__.py b/verdict-backend/app/models/__init__.py index 990fd0e..fe342dd 100644 --- a/verdict-backend/app/models/__init__.py +++ b/verdict-backend/app/models/__init__.py @@ -1,5 +1,2 @@ -from app.models.asset import Asset -from app.models.base import GoldSourceMixin, TagsMixin, TimestampMixin -from app.models.gold_source import GoldSourceType - -__all__ = ["Asset", "GoldSourceMixin", "GoldSourceType", "TagsMixin", "TimestampMixin"] +import app.models.asset # register for alembic autogenerate +import app.models.system # noqa: F401 # register for alembic autogenerate diff --git a/verdict-backend/app/models/asset.py b/verdict-backend/app/models/asset.py index 0f1de13..701794d 100644 --- a/verdict-backend/app/models/asset.py +++ b/verdict-backend/app/models/asset.py @@ -1,10 +1,33 @@ -from sqlmodel import Field +from typing import TYPE_CHECKING -from app.models.base import BaseModel, GoldSourceMixin, TagsMixin, TimestampMixin +from sqlmodel import Field, Relationship, SQLModel +from app.models.base import BaseModel, GoldSourceMixin, PublicModel, TagsMixin + +if TYPE_CHECKING: + from app.models.system import System -class Asset(TimestampMixin, GoldSourceMixin, TagsMixin, BaseModel, table=True): - __tablename__ = "asset" # pyright: ignore[reportAssignmentType] # SQLAlchemy declared_attr +class AssetBase(GoldSourceMixin, TagsMixin, SQLModel): name: str = Field(nullable=False) description: str = Field(default="", nullable=False) + + +class Asset(AssetBase, BaseModel, table=True): + __tablename__ = "asset" # pyright: ignore[reportAssignmentType] # SQLAlchemy declared_attr + + systems: list["System"] = Relationship( + sa_relationship_kwargs={ + "secondary": "asset_system", + "lazy": "selectin", + "viewonly": True, + }, + ) + + +class AssetCreate(AssetBase): + pass + + +class AssetPublic(AssetBase, PublicModel): + pass diff --git a/verdict-backend/app/models/base.py b/verdict-backend/app/models/base.py index 9ccc258..dfa4fef 100644 --- a/verdict-backend/app/models/base.py +++ b/verdict-backend/app/models/base.py @@ -1,6 +1,5 @@ -from app.models.base_model import BaseModel +from app.models.base_model import BaseModel, PublicModel from app.models.gold_source import GoldSourceMixin from app.models.tags import TagsMixin -from app.models.timestamp import TimestampMixin -__all__ = ["BaseModel", "GoldSourceMixin", "TagsMixin", "TimestampMixin"] +__all__ = ["BaseModel", "GoldSourceMixin", "PublicModel", "TagsMixin"] diff --git a/verdict-backend/app/models/base_model.py b/verdict-backend/app/models/base_model.py index 43e6351..5a67308 100644 --- a/verdict-backend/app/models/base_model.py +++ b/verdict-backend/app/models/base_model.py @@ -1,7 +1,33 @@ +from datetime import datetime +from typing import Any, cast + +import sqlalchemy as sa from sqlmodel import Field, SQLModel class BaseModel(SQLModel): - """Base for all table models. Provides an auto-increment integer primary key.""" + """Base for all table models. Provides an auto-increment primary key and timestamps.""" id: int | None = Field(default=None, primary_key=True) + created_at: datetime | None = Field( + default=None, + sa_type=cast("type[Any]", sa.DateTime(timezone=True)), + sa_column_kwargs={"server_default": sa.func.now(), "nullable": False}, + ) + updated_at: datetime | None = Field( + default=None, + sa_type=cast("type[Any]", sa.DateTime(timezone=True)), + sa_column_kwargs={ + "server_default": sa.func.now(), + "onupdate": sa.func.clock_timestamp(), + "nullable": False, + }, + ) + + +class PublicModel(SQLModel): + """Mixin for public/read models. Provides narrowed id and timestamps.""" + + id: int + created_at: datetime + updated_at: datetime diff --git a/verdict-backend/app/models/gold_source.py b/verdict-backend/app/models/gold_source.py index 28894a1..18f245b 100644 --- a/verdict-backend/app/models/gold_source.py +++ b/verdict-backend/app/models/gold_source.py @@ -1,16 +1,18 @@ from enum import StrEnum +from typing import Any, overload import sqlalchemy as sa from sqlalchemy.exc import OperationalError from sqlmodel import Field, Session, SQLModel, select from app.errors import DBError, db_error_from -from app.models.base_model import BaseModel +from app.models.base_model import BaseModel, PublicModel from app.result import Err, Nothing, Ok, Option, Result, Some class GoldSourceType(StrEnum): ASSET_INVENTORY = "asset-inventory" + CMDB = "cmdb" class GoldSourceMixin(SQLModel): @@ -26,16 +28,42 @@ class GoldSourceMixin(SQLModel): gold_source_type: str = Field(nullable=False) @staticmethod + @overload def get_by_gold_source[T: BaseModel]( session: Session, model_class: type[T], gold_source_type: str, gold_source_id: str, - ) -> Result[Option[T], DBError]: + ) -> Result[Option[T], DBError]: ... + + @staticmethod + @overload + def get_by_gold_source[T: BaseModel, P: PublicModel]( + session: Session, + model_class: type[T], + gold_source_type: str, + gold_source_id: str, + *, + public_class: type[P], + ) -> Result[Option[P], DBError]: ... + + @staticmethod + def get_by_gold_source[T: BaseModel, P: PublicModel]( + session: Session, + model_class: type[T], + gold_source_type: str, + gold_source_id: str, + *, + public_class: type[P] | None = None, + ) -> Result[Option[Any], DBError]: """Query a model by its external gold source reference. Returns ``Ok(Some(record))`` on success, ``Ok(Nothing())`` when no record matches, or ``Err(DBError(...))`` on database operational errors. + + When *public_class* is provided, the record is validated into that type + before being returned, narrowing optional fields like ``id`` and + timestamps. """ if not issubclass(model_class, GoldSourceMixin): raise TypeError(f"{model_class.__name__} does not use GoldSourceMixin") @@ -54,4 +82,7 @@ def get_by_gold_source[T: BaseModel]( if record is None: return Ok(Nothing()) + if public_class is not None: + return Ok(Some(public_class.model_validate(record, from_attributes=True))) + return Ok(Some(record)) diff --git a/verdict-backend/app/models/system.py b/verdict-backend/app/models/system.py new file mode 100644 index 0000000..454c45d --- /dev/null +++ b/verdict-backend/app/models/system.py @@ -0,0 +1,36 @@ +from typing import TYPE_CHECKING + +import sqlalchemy as sa +from sqlmodel import Field, Relationship, SQLModel + +from app.models.base import BaseModel, GoldSourceMixin, PublicModel, TagsMixin + +if TYPE_CHECKING: + from app.models.asset import Asset + +asset_system = sa.Table( + "asset_system", + BaseModel.metadata, # pyright: ignore[reportAttributeAccessIssue] # SQLModel exposes metadata via SQLAlchemy + sa.Column("asset_id", sa.Integer, sa.ForeignKey("asset.id"), primary_key=True), + sa.Column("system_id", sa.Integer, sa.ForeignKey("system.id"), primary_key=True), +) + + +class SystemBase(GoldSourceMixin, TagsMixin, SQLModel): + primary_fqdn: str = Field(nullable=False) + + +class System(SystemBase, BaseModel, table=True): + __tablename__ = "system" # pyright: ignore[reportAssignmentType] # SQLAlchemy declared_attr + + assets: list["Asset"] = Relationship( + sa_relationship_kwargs={"secondary": asset_system, "lazy": "selectin"}, + ) + + +class SystemCreate(SystemBase): + pass + + +class SystemPublic(SystemBase, PublicModel): + asset_ids: list[int] = [] diff --git a/verdict-backend/app/models/timestamp.py b/verdict-backend/app/models/timestamp.py deleted file mode 100644 index e995f5e..0000000 --- a/verdict-backend/app/models/timestamp.py +++ /dev/null @@ -1,22 +0,0 @@ -from datetime import datetime -from typing import Any, cast - -import sqlalchemy as sa -from sqlmodel import Field, SQLModel - - -class TimestampMixin(SQLModel): - created_at: datetime | None = Field( - default=None, - sa_type=cast("type[Any]", sa.DateTime(timezone=True)), - sa_column_kwargs={"server_default": sa.func.now(), "nullable": False}, - ) - updated_at: datetime | None = Field( - default=None, - sa_type=cast("type[Any]", sa.DateTime(timezone=True)), - sa_column_kwargs={ - "server_default": sa.func.now(), - "onupdate": sa.func.clock_timestamp(), - "nullable": False, - }, - ) diff --git a/verdict-backend/app/queries.py b/verdict-backend/app/queries.py index fe20714..9f83922 100644 --- a/verdict-backend/app/queries.py +++ b/verdict-backend/app/queries.py @@ -1,11 +1,11 @@ from dataclasses import dataclass -from typing import final +from typing import Any, final, overload from sqlalchemy.exc import OperationalError from sqlmodel import Session, func, select from app.errors import DBError, db_error_from -from app.models.base_model import BaseModel +from app.models.base_model import BaseModel, PublicModel from app.result import Err, Nothing, Ok, Option, Result, Some @@ -16,11 +16,31 @@ class PaginatedResult[T]: total: int +@overload def get_by_id[T: BaseModel]( session: Session, model_class: type[T], record_id: int, -) -> Result[Option[T], DBError]: +) -> Result[Option[T], DBError]: ... + + +@overload +def get_by_id[T: BaseModel, P: PublicModel]( + session: Session, + model_class: type[T], + record_id: int, + *, + public_class: type[P], +) -> Result[Option[P], DBError]: ... + + +def get_by_id[T: BaseModel, P: PublicModel]( + session: Session, + model_class: type[T], + record_id: int, + *, + public_class: type[P] | None = None, +) -> Result[Option[Any], DBError]: try: record = session.get(model_class, record_id) except OperationalError as e: @@ -28,15 +48,41 @@ def get_by_id[T: BaseModel]( if record is None: return Ok(Nothing()) + + if public_class is not None: + return Ok(Some(public_class.model_validate(record, from_attributes=True))) + return Ok(Some(record)) +@overload def get_paginated[T: BaseModel]( session: Session, model_class: type[T], offset: int, limit: int, -) -> Result[PaginatedResult[T], DBError]: +) -> Result[PaginatedResult[T], DBError]: ... + + +@overload +def get_paginated[T: BaseModel, P: PublicModel]( + session: Session, + model_class: type[T], + offset: int, + limit: int, + *, + public_class: type[P], +) -> Result[PaginatedResult[P], DBError]: ... + + +def get_paginated[T: BaseModel, P: PublicModel]( + session: Session, + model_class: type[T], + offset: int, + limit: int, + *, + public_class: type[P] | None = None, +) -> Result[PaginatedResult[Any], DBError]: try: total = session.exec(select(func.count()).select_from(model_class)).one() items = list( @@ -47,4 +93,8 @@ def get_paginated[T: BaseModel]( except OperationalError as e: return Err(db_error_from(e)) + if public_class is not None: + public_items = [public_class.model_validate(i, from_attributes=True) for i in items] + return Ok(PaginatedResult(items=public_items, total=total)) + return Ok(PaginatedResult(items=items, total=total)) diff --git a/verdict-backend/app/routes/assets.py b/verdict-backend/app/routes/assets.py index 01f65d5..810e15e 100644 --- a/verdict-backend/app/routes/assets.py +++ b/verdict-backend/app/routes/assets.py @@ -1,16 +1,12 @@ -import httpx from fastapi import APIRouter, Depends, Query from sqlmodel import Session -from app.config import settings from app.db import get_session -from app.deps import get_http_client -from app.models.asset import Asset +from app.models.asset import Asset, AssetPublic from app.models.gold_source import GoldSourceMixin from app.queries import get_by_id, get_paginated from app.result import unwrap_optional_or_raise, unwrap_or_raise -from app.schemas.asset import AssetListResponse, AssetResponse, IngestionResponse -from app.services.asset_ingestion import ingest_assets +from app.schemas.asset import AssetListResponse router = APIRouter(prefix="/assets", tags=["assets"]) @@ -21,42 +17,28 @@ def list_assets( offset: int = Query(default=0, ge=0), limit: int = Query(default=50, ge=1, le=200), ) -> AssetListResponse: - paginated = unwrap_or_raise(get_paginated(session, Asset, offset, limit)) - return AssetListResponse( - assets=[AssetResponse.model_validate(a, from_attributes=True) for a in paginated.items], - total=paginated.total, + paginated = unwrap_or_raise( + get_paginated(session, Asset, offset, limit, public_class=AssetPublic) ) + return AssetListResponse(assets=paginated.items, total=paginated.total) @router.get( "/by-gold-source/{source_type}/{source_id}", - response_model=AssetResponse, + response_model=AssetPublic, ) def get_asset_by_gold_source( source_type: str, source_id: str, session: Session = Depends(get_session), -) -> AssetResponse: - result = GoldSourceMixin.get_by_gold_source(session, Asset, source_type, source_id) - asset = unwrap_optional_or_raise(result, not_found_detail="Asset not found") - return AssetResponse.model_validate(asset, from_attributes=True) - - -@router.get("/{asset_id}", response_model=AssetResponse) -def get_asset(asset_id: int, session: Session = Depends(get_session)) -> AssetResponse: - result = get_by_id(session, Asset, asset_id) - asset = unwrap_optional_or_raise(result, not_found_detail="Asset not found") - return AssetResponse.model_validate(asset, from_attributes=True) +) -> AssetPublic: + result = GoldSourceMixin.get_by_gold_source( + session, Asset, source_type, source_id, public_class=AssetPublic + ) + return unwrap_optional_or_raise(result, not_found_detail="Asset not found") -@router.post("/ingest", response_model=IngestionResponse) -def trigger_ingestion( - session: Session = Depends(get_session), - client: httpx.Client = Depends(get_http_client), -) -> IngestionResponse: - assets = unwrap_or_raise( - ingest_assets(session, client, settings.asset_inventory_url), - err_status=502, - ) - session.commit() - return IngestionResponse(ingested=len(assets)) +@router.get("/{asset_id}", response_model=AssetPublic) +def get_asset(asset_id: int, session: Session = Depends(get_session)) -> AssetPublic: + result = get_by_id(session, Asset, asset_id, public_class=AssetPublic) + return unwrap_optional_or_raise(result, not_found_detail="Asset not found") diff --git a/verdict-backend/app/routes/ingestion.py b/verdict-backend/app/routes/ingestion.py new file mode 100644 index 0000000..f519108 --- /dev/null +++ b/verdict-backend/app/routes/ingestion.py @@ -0,0 +1,33 @@ +import httpx +from fastapi import APIRouter, Depends +from sqlmodel import Session + +from app.config import settings +from app.db import get_session +from app.deps import get_http_client +from app.result import unwrap_or_raise +from app.schemas.ingestion import FullIngestionResponse +from app.services.asset_ingestion import ingest_assets +from app.services.cmdb_ingestion import ingest_systems + +router = APIRouter(tags=["ingestion"]) + + +@router.post("/ingest", response_model=FullIngestionResponse) +def trigger_full_ingestion( + session: Session = Depends(get_session), + client: httpx.Client = Depends(get_http_client), +) -> FullIngestionResponse: + assets = unwrap_or_raise( + ingest_assets(session, client, settings.asset_inventory_url), + err_status=502, + ) + systems = unwrap_or_raise( + ingest_systems(session, client, settings.cmdb_url), + err_status=502, + ) + session.commit() + return FullIngestionResponse( + assets_ingested=len(assets), + systems_ingested=len(systems), + ) diff --git a/verdict-backend/app/routes/systems.py b/verdict-backend/app/routes/systems.py new file mode 100644 index 0000000..ffd95bb --- /dev/null +++ b/verdict-backend/app/routes/systems.py @@ -0,0 +1,51 @@ +from fastapi import APIRouter, Depends, Query +from sqlmodel import Session + +from app.db import get_session +from app.models.gold_source import GoldSourceMixin +from app.models.system import System, SystemPublic +from app.queries import get_by_id, get_paginated +from app.result import unwrap_optional_or_raise, unwrap_or_raise +from app.schemas.system import SystemListResponse + +router = APIRouter(prefix="/systems", tags=["systems"]) + + +def _to_public(system: System) -> SystemPublic: + public = SystemPublic.model_validate(system, from_attributes=True) + public.asset_ids = [a.id for a in system.assets if a.id is not None] + return public + + +@router.get("/", response_model=SystemListResponse) +def list_systems( + session: Session = Depends(get_session), + offset: int = Query(default=0, ge=0), + limit: int = Query(default=50, ge=1, le=200), +) -> SystemListResponse: + paginated = unwrap_or_raise(get_paginated(session, System, offset, limit)) + return SystemListResponse( + systems=[_to_public(s) for s in paginated.items], + total=paginated.total, + ) + + +@router.get( + "/by-gold-source/{source_type}/{source_id}", + response_model=SystemPublic, +) +def get_system_by_gold_source( + source_type: str, + source_id: str, + session: Session = Depends(get_session), +) -> SystemPublic: + result = GoldSourceMixin.get_by_gold_source(session, System, source_type, source_id) + system = unwrap_optional_or_raise(result, not_found_detail="System not found") + return _to_public(system) + + +@router.get("/{system_id}", response_model=SystemPublic) +def get_system(system_id: int, session: Session = Depends(get_session)) -> SystemPublic: + result = get_by_id(session, System, system_id) + system = unwrap_optional_or_raise(result, not_found_detail="System not found") + return _to_public(system) diff --git a/verdict-backend/app/schemas/asset.py b/verdict-backend/app/schemas/asset.py index 0aaf7b9..d2a0e37 100644 --- a/verdict-backend/app/schemas/asset.py +++ b/verdict-backend/app/schemas/asset.py @@ -1,18 +1,8 @@ from pydantic import BaseModel -from app.schemas.base import GoldSourceResponseMixin, TagsResponseMixin, TimestampResponseMixin - - -class AssetResponse(TimestampResponseMixin, GoldSourceResponseMixin, TagsResponseMixin): - id: int - name: str - description: str +from app.models.asset import AssetPublic class AssetListResponse(BaseModel): - assets: list[AssetResponse] + assets: list[AssetPublic] total: int - - -class IngestionResponse(BaseModel): - ingested: int diff --git a/verdict-backend/app/schemas/base.py b/verdict-backend/app/schemas/base.py deleted file mode 100644 index 48976f7..0000000 --- a/verdict-backend/app/schemas/base.py +++ /dev/null @@ -1,17 +0,0 @@ -from datetime import datetime - -from pydantic import BaseModel - - -class TimestampResponseMixin(BaseModel): - created_at: datetime - updated_at: datetime - - -class GoldSourceResponseMixin(BaseModel): - gold_source_id: str - gold_source_type: str - - -class TagsResponseMixin(BaseModel): - tags: list[str] diff --git a/verdict-backend/app/schemas/external/cmdb.py b/verdict-backend/app/schemas/external/cmdb.py new file mode 100644 index 0000000..0742fd8 --- /dev/null +++ b/verdict-backend/app/schemas/external/cmdb.py @@ -0,0 +1,13 @@ +from pydantic import BaseModel + + +class SystemIndexItem(BaseModel): + id: str + primary_fqdn: str + + +class SystemDetail(BaseModel): + id: str + primary_fqdn: str + asset_gold_source_ids: list[str] + tags: list[str] diff --git a/verdict-backend/app/schemas/ingestion.py b/verdict-backend/app/schemas/ingestion.py new file mode 100644 index 0000000..450de33 --- /dev/null +++ b/verdict-backend/app/schemas/ingestion.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class FullIngestionResponse(BaseModel): + assets_ingested: int + systems_ingested: int diff --git a/verdict-backend/app/schemas/system.py b/verdict-backend/app/schemas/system.py new file mode 100644 index 0000000..c56c091 --- /dev/null +++ b/verdict-backend/app/schemas/system.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel + +from app.models.system import SystemPublic + + +class SystemListResponse(BaseModel): + systems: list[SystemPublic] + total: int diff --git a/verdict-backend/app/services/asset_ingestion.py b/verdict-backend/app/services/asset_ingestion.py index bd55dd1..56d72c4 100644 --- a/verdict-backend/app/services/asset_ingestion.py +++ b/verdict-backend/app/services/asset_ingestion.py @@ -8,7 +8,7 @@ from sqlmodel import Session from app.errors import DBError, FetchError, IngestionError, ValidationError, db_error_from -from app.models.asset import Asset +from app.models.asset import Asset, AssetCreate, AssetPublic from app.models.gold_source import GoldSourceMixin, GoldSourceType from app.result import Err, Ok, Result, Some from app.schemas.external.asset_inventory import AssetDetail, AssetIndexItem @@ -25,7 +25,6 @@ def fetch_index( client: httpx.Client, url: str, ) -> Result[list[AssetIndexItem], SourceError]: - """Fetch the asset index from the external inventory.""" try: response = client.get(url) response.raise_for_status() @@ -46,7 +45,6 @@ def fetch_detail( url: str, item_id: str, ) -> Result[AssetDetail, SourceError]: - """Fetch a single asset's detail from the external inventory.""" detail_url = f"{url}/{quote(item_id, safe='')}" try: response = client.get(detail_url) @@ -63,9 +61,8 @@ def fetch_detail( return Ok(detail) -def to_asset(detail: AssetDetail) -> Asset: - """Convert an external asset detail to a verdict Asset.""" - return Asset( +def to_asset(detail: AssetDetail) -> AssetCreate: + return AssetCreate( name=detail.name, description=detail.description, tags=detail.tags, @@ -78,24 +75,19 @@ def ingest_assets( session: Session, client: httpx.Client, url: str, -) -> Result[list[Asset], IngestionError]: - """Fetch assets from the external inventory and upsert them. - - Three phases: fetch index, fetch detail for each, convert and upsert. - Does not commit — the caller owns the session lifecycle. - """ +) -> Result[list[AssetPublic], IngestionError]: index_result = fetch_index(client, url) if isinstance(index_result, Err): return Err(index_result.value) - assets: list[Asset] = [] + assets: list[AssetPublic] = [] for index_item in index_result.value: detail_result = fetch_detail(client, url, index_item.id) if isinstance(detail_result, Err): return Err(detail_result.value) - asset = to_asset(detail_result.value) - upsert_result = _upsert_asset(session, asset) + asset_create = to_asset(detail_result.value) + upsert_result = _upsert_asset(session, asset_create) if isinstance(upsert_result, Err): return Err(upsert_result.value) assets.append(upsert_result.value) @@ -109,25 +101,33 @@ def ingest_assets( def _upsert_asset( session: Session, - asset: Asset, -) -> Result[Asset, DBError]: - """Look up an existing asset by gold source or create a new one.""" + asset_create: AssetCreate, +) -> Result[AssetPublic, DBError]: result = GoldSourceMixin.get_by_gold_source( session, Asset, - asset.gold_source_type, - asset.gold_source_id, + asset_create.gold_source_type, + asset_create.gold_source_id, ) if isinstance(result, Err): - return result + return Err(result.value) match result.value: case Some(existing): - existing.name = asset.name - existing.description = asset.description - existing.tags = asset.tags + existing.name = asset_create.name + existing.description = asset_create.description + existing.tags = asset_create.tags session.add(existing) - return Ok(existing) + try: + session.flush() + except OperationalError as e: + return Err(db_error_from(e)) + return Ok(AssetPublic.model_validate(existing, from_attributes=True)) case _: - session.add(asset) - return Ok(asset) + new = Asset.model_validate(asset_create, from_attributes=True) + session.add(new) + try: + session.flush() + except OperationalError as e: + return Err(db_error_from(e)) + return Ok(AssetPublic.model_validate(new, from_attributes=True)) diff --git a/verdict-backend/app/services/cmdb_ingestion.py b/verdict-backend/app/services/cmdb_ingestion.py new file mode 100644 index 0000000..1aa7151 --- /dev/null +++ b/verdict-backend/app/services/cmdb_ingestion.py @@ -0,0 +1,206 @@ +import logging +from urllib.parse import quote + +import httpx +import sqlalchemy as sa +from pydantic import TypeAdapter +from pydantic import ValidationError as PydanticValidationError +from sqlalchemy.exc import OperationalError +from sqlmodel import Session + +from app.errors import DBError, FetchError, IngestionError, ValidationError, db_error_from +from app.models.asset import Asset, AssetPublic +from app.models.gold_source import GoldSourceMixin, GoldSourceType +from app.models.system import System, SystemCreate, SystemPublic, asset_system +from app.result import Err, Nothing, Ok, Result, Some +from app.schemas.external.cmdb import SystemDetail, SystemIndexItem + +logger = logging.getLogger(__name__) + +_index_adapter = TypeAdapter(list[SystemIndexItem]) +_detail_adapter = TypeAdapter(SystemDetail) + +type SourceError = FetchError | ValidationError + + +def fetch_index( + client: httpx.Client, + url: str, +) -> Result[list[SystemIndexItem], SourceError]: + try: + response = client.get(url) + response.raise_for_status() + except httpx.HTTPError as e: + return Err(FetchError(url=url, raw=str(e))) + + try: + items = _index_adapter.validate_json(response.content) + except PydanticValidationError as e: + logger.error("Schema mismatch from CMDB index: %s", e) + return Err(ValidationError(raw=str(e))) + + return Ok(items) + + +def fetch_detail( + client: httpx.Client, + url: str, + item_id: str, +) -> Result[SystemDetail, SourceError]: + detail_url = f"{url}/{quote(item_id, safe='')}" + try: + response = client.get(detail_url) + response.raise_for_status() + except httpx.HTTPError as e: + return Err(FetchError(url=detail_url, raw=str(e))) + + try: + detail = _detail_adapter.validate_json(response.content) + except PydanticValidationError as e: + logger.error("Schema mismatch from CMDB detail %s: %s", item_id, e) + return Err(ValidationError(raw=str(e))) + + return Ok(detail) + + +def to_system(detail: SystemDetail) -> SystemCreate: + return SystemCreate( + primary_fqdn=detail.primary_fqdn, + tags=detail.tags, + gold_source_id=detail.id, + gold_source_type=GoldSourceType.CMDB, + ) + + +def _resolve_asset_ids( + session: Session, + gold_source_ids: list[str], +) -> Result[list[int], ValidationError | DBError]: + asset_ids: list[int] = [] + for gs_id in gold_source_ids: + result = GoldSourceMixin.get_by_gold_source( + session, + Asset, + GoldSourceType.ASSET_INVENTORY, + gs_id, + public_class=AssetPublic, + ) + if isinstance(result, Err): + return Err(result.value) + match result.value: + case Some(asset): + asset_ids.append(asset.id) + case Nothing(): + return Err( + ValidationError( + raw=f"unresolvable asset reference: {gs_id}", + ) + ) + return Ok(asset_ids) + + +def _upsert_system( + session: Session, + system_create: SystemCreate, +) -> Result[SystemPublic, DBError]: + result = GoldSourceMixin.get_by_gold_source( + session, + System, + system_create.gold_source_type, + system_create.gold_source_id, + ) + if isinstance(result, Err): + return Err(result.value) + + match result.value: + case Some(existing): + existing.primary_fqdn = system_create.primary_fqdn + existing.tags = system_create.tags + session.add(existing) + try: + session.flush() + except OperationalError as e: + return Err(db_error_from(e)) + return Ok(SystemPublic.model_validate(existing, from_attributes=True)) + case _: + new = System.model_validate(system_create, from_attributes=True) + session.add(new) + try: + session.flush() + except OperationalError as e: + return Err(db_error_from(e)) + return Ok(SystemPublic.model_validate(new, from_attributes=True)) + + +def _sync_asset_links( + session: Session, + system_id: int, + desired_asset_ids: list[int], +) -> Result[None, DBError]: + try: + existing_rows = session.execute( + sa.select(asset_system.c.asset_id).where(asset_system.c.system_id == system_id) + ).all() + existing_ids = {row.asset_id for row in existing_rows} + desired_ids = set(desired_asset_ids) + + to_remove = existing_ids - desired_ids + to_add = desired_ids - existing_ids + + if to_remove: + session.execute( + sa.delete(asset_system).where( + asset_system.c.system_id == system_id, + asset_system.c.asset_id.in_(to_remove), + ) + ) + for aid in to_add: + session.execute(sa.insert(asset_system).values(system_id=system_id, asset_id=aid)) + except OperationalError as e: + return Err(db_error_from(e)) + return Ok(None) + + +def ingest_systems( + session: Session, + client: httpx.Client, + url: str, +) -> Result[list[SystemPublic], IngestionError]: + index_result = fetch_index(client, url) + if isinstance(index_result, Err): + return Err(index_result.value) + + details: list[SystemDetail] = [] + for index_item in index_result.value: + detail_result = fetch_detail(client, url, index_item.id) + if isinstance(detail_result, Err): + return Err(detail_result.value) + details.append(detail_result.value) + + resolved_assets: dict[str, list[int]] = {} + for detail in details: + if detail.asset_gold_source_ids: + resolve_result = _resolve_asset_ids(session, detail.asset_gold_source_ids) + if isinstance(resolve_result, Err): + return Err(resolve_result.value) + resolved_assets[detail.id] = resolve_result.value + else: + resolved_assets[detail.id] = [] + + systems: list[SystemPublic] = [] + for detail in details: + system_create = to_system(detail) + upsert_result = _upsert_system(session, system_create) + if isinstance(upsert_result, Err): + return Err(upsert_result.value) + system_public = upsert_result.value + sync_result = _sync_asset_links(session, system_public.id, resolved_assets[detail.id]) + if isinstance(sync_result, Err): + return Err(sync_result.value) + systems.append(system_public) + + try: + session.flush() + except OperationalError as e: + return Err(db_error_from(e)) + return Ok(systems) diff --git a/verdict-backend/devenv.lock b/verdict-backend/devenv.lock index a9a3bb6..9ce1c82 100644 --- a/verdict-backend/devenv.lock +++ b/verdict-backend/devenv.lock @@ -3,10 +3,11 @@ "devenv": { "locked": { "dir": "src/modules", - "lastModified": 1771157881, + "lastModified": 1773077419, + "narHash": "sha256-Vr3Zrg4DEm4xy0StawiTEUs8TXruj1WdRZyDhlq1aPg=", "owner": "cachix", "repo": "devenv", - "rev": "b0b3dfa70ec90fa49f672e579f186faf4f61bd4b", + "rev": "2105b1a0272e32b0d6a9e213b1c381ab4ae4a692", "type": "github" }, "original": { @@ -20,6 +21,7 @@ "flake": false, "locked": { "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", "owner": "NixOS", "repo": "flake-compat", "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", @@ -34,10 +36,11 @@ "flake-compat_2": { "flake": false, "locked": { - "lastModified": 1767039857, + "lastModified": 1747046372, + "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=", "owner": "edolstra", "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885", "type": "github" }, "original": { @@ -50,15 +53,14 @@ "inputs": { "flake-compat": "flake-compat", "gitignore": "gitignore", - "nixpkgs": [ - "nixpkgs" - ] + "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1770726378, + "lastModified": 1772893680, + "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "5eaaedde414f6eb1aea8b8525c466dc37bba95ae", + "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9", "type": "github" }, "original": { @@ -75,10 +77,11 @@ ] }, "locked": { - "lastModified": 1762808025, + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", "owner": "hercules-ci", "repo": "gitignore.nix", - "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", "type": "github" }, "original": { @@ -88,20 +91,18 @@ } }, "nixpkgs": { - "inputs": { - "nixpkgs-src": "nixpkgs-src" - }, "locked": { - "lastModified": 1770434727, - "owner": "cachix", - "repo": "devenv-nixpkgs", - "rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348", + "lastModified": 1770073757, + "narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "47472570b1e607482890801aeaf29bfb749884f6", "type": "github" }, "original": { - "owner": "cachix", - "ref": "rolling", - "repo": "devenv-nixpkgs", + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", "type": "github" } }, @@ -113,10 +114,11 @@ ] }, "locked": { - "lastModified": 1770193939, + "lastModified": 1772559541, + "narHash": "sha256-1cWCYOnOz3K7FchI9IbIbu1qhi68mSNZSuYlE5jEv1I=", "owner": "cachix", "repo": "nixpkgs-python", - "rev": "0630618bfe33895453257fb606af75aa71247393", + "rev": "188ffc273f679a2d95a1c8aba0be083545417bdd", "type": "github" }, "original": { @@ -128,11 +130,11 @@ "nixpkgs-src": { "flake": false, "locked": { - "lastModified": 1769922788, - "narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=", + "lastModified": 1772173633, + "narHash": "sha256-MOH58F4AIbCkh6qlQcwMycyk5SWvsqnS/TCfnqDlpj4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "207d15f1a6603226e1e223dc79ac29c7846da32e", + "rev": "c0f3d81a7ddbc2b1332be0d8481a672b4f6004d6", "type": "github" }, "original": { @@ -142,18 +144,34 @@ "type": "github" } }, + "nixpkgs_2": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1772749504, + "narHash": "sha256-eqtQIz0alxkQPym+Zh/33gdDjkkch9o6eHnMPnXFXN0=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "08543693199362c1fddb8f52126030d0d374ba2e", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "devenv": "devenv", "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs", - "nixpkgs-python": "nixpkgs-python", - "pre-commit-hooks": [ - "git-hooks" - ] + "nixpkgs": "nixpkgs_2", + "nixpkgs-python": "nixpkgs-python" } } }, "root": "root", "version": 7 -} +} \ No newline at end of file diff --git a/verdict-backend/devenv.nix b/verdict-backend/devenv.nix index 654eaad..101b6ba 100644 --- a/verdict-backend/devenv.nix +++ b/verdict-backend/devenv.nix @@ -14,6 +14,8 @@ let postgres_password = "postgres"; in { + process.manager.implementation = "process-compose"; + packages = with pkgs; [ pgcli ]; @@ -103,6 +105,7 @@ in DATABASE_PASSWORD=${postgres_password} DATABASE_URL=postgresql://${postgres_user}:${postgres_password}@${postgres_host}:${toString postgres_port}/${database_name} ASSET_INVENTORY_URL=http://localhost:4010/assets + CMDB_URL=http://localhost:4011/systems EOF echo "Generated .env.sample" ''; @@ -135,36 +138,54 @@ in just db-test-reset DATABASE_NAME=${database_name}_test just db-migrate just check - DATABASE_NAME=${database_name}_test uv run pytest --disable-plugin-autoload -p asyncio -m 'not e2e' + DATABASE_NAME=${database_name}_test uv run pytest --disable-plugin-autoload -p asyncio -m 'not e2e' --junit-xml=test-results/unit.xml # Cleanup background processes on any exit - MOCK_PID= + ASSET_MOCK_PID= + CMDB_MOCK_PID= APP_PID= cleanup() { - kill $APP_PID $MOCK_PID 2>/dev/null || true - wait $APP_PID $MOCK_PID 2>/dev/null || true + kill $APP_PID $ASSET_MOCK_PID $CMDB_MOCK_PID 2>/dev/null || true + wait $APP_PID $ASSET_MOCK_PID $CMDB_MOCK_PID 2>/dev/null || true } trap cleanup EXIT INT TERM - # Start mock service + # Start mock services echo "Starting mock asset inventory..." (cd $MOCK_SERVICES_DIR && exec uv run uvicorn asset_inventory.app:app --host 0.0.0.0 --port 4010) & - MOCK_PID=$! + ASSET_MOCK_PID=$! + + echo "Starting mock CMDB..." + (cd $MOCK_SERVICES_DIR && exec uv run uvicorn cmdb.app:app --host 0.0.0.0 --port 4011) & + CMDB_MOCK_PID=$! retries=0 until curl -sf http://localhost:4010/assets > /dev/null 2>&1; do retries=$((retries + 1)) if [ $retries -ge 30 ]; then - echo "ERROR: Mock service failed to start" + echo "ERROR: Asset inventory mock failed to start" + exit 1 + fi + sleep 1 + done + echo "Asset inventory mock ready" + + retries=0 + until curl -sf http://localhost:4011/systems > /dev/null 2>&1; do + retries=$((retries + 1)) + if [ $retries -ge 30 ]; then + echo "ERROR: CMDB mock failed to start" exit 1 fi sleep 1 done - echo "Mock service ready" + echo "CMDB mock ready" # Start verdict app against test DB echo "Starting verdict app..." - (DATABASE_NAME=${database_name}_test ASSET_INVENTORY_URL=http://localhost:4010/assets \ + (DATABASE_NAME=${database_name}_test \ + ASSET_INVENTORY_URL=http://localhost:4010/assets \ + CMDB_URL=http://localhost:4011/systems \ exec uv run uvicorn app.main:app --host 0.0.0.0 --port 8000) & APP_PID=$! @@ -182,6 +203,6 @@ in # Reset test DB and run E2E tests just db-test-reset DATABASE_NAME=${database_name}_test just db-migrate - just test-e2e + DATABASE_NAME=${database_name}_test uv run pytest --disable-plugin-autoload -p asyncio -m e2e --junit-xml=test-results/e2e.xml ''; } diff --git a/verdict-backend/devenv.yaml b/verdict-backend/devenv.yaml index 760c484..9f21d10 100644 --- a/verdict-backend/devenv.yaml +++ b/verdict-backend/devenv.yaml @@ -9,3 +9,5 @@ inputs: inputs: nixpkgs: follows: nixpkgs + git-hooks: + url: github:cachix/git-hooks.nix diff --git a/verdict-backend/justfile b/verdict-backend/justfile index 90d49f0..f97591d 100644 --- a/verdict-backend/justfile +++ b/verdict-backend/justfile @@ -9,9 +9,13 @@ default: dev: uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 -# Run tests +# Start the FastAPI development server against the test database +dev-test: + DATABASE_NAME=$DATABASE_NAME_TEST uv run uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 + +# Run tests (excludes e2e; use test-e2e for those) test *ARGS: - DATABASE_NAME=$DATABASE_NAME_TEST uv run pytest --disable-plugin-autoload -p asyncio {{ARGS}} + DATABASE_NAME=$DATABASE_NAME_TEST uv run pytest --disable-plugin-autoload -p asyncio -m "not e2e" {{ARGS}} # Run end-to-end tests (requires services to be running) test-e2e *ARGS: diff --git a/verdict-backend/tests/e2e/conftest.py b/verdict-backend/tests/e2e/conftest.py index 1b7b6bd..5c4d0c7 100644 --- a/verdict-backend/tests/e2e/conftest.py +++ b/verdict-backend/tests/e2e/conftest.py @@ -4,6 +4,26 @@ import httpx import pytest +import sqlalchemy as sa + +from app.db import engine +from app.models.asset import Asset +from app.models.system import System, asset_system + + +@pytest.fixture(scope="session", autouse=True) +def _clean_db() -> Generator[None, None, None]: + """Truncate all tables before and after the e2e session.""" + _truncate() + yield + _truncate() + + +def _truncate() -> None: + with engine.begin() as conn: + conn.execute(sa.delete(asset_system)) + conn.execute(sa.delete(System)) + conn.execute(sa.delete(Asset)) @pytest.fixture(scope="session") diff --git a/verdict-backend/tests/e2e/test_asset_ingestion.py b/verdict-backend/tests/e2e/test_asset_ingestion.py index 49dc7f7..b42fecc 100644 --- a/verdict-backend/tests/e2e/test_asset_ingestion.py +++ b/verdict-backend/tests/e2e/test_asset_ingestion.py @@ -3,6 +3,7 @@ These tests assume all services are running: - PostgreSQL on localhost:5432 - Mock Asset Inventory on localhost:4010 +- Mock CMDB on localhost:4011 - Verdict API on localhost:8000 """ @@ -13,13 +14,11 @@ def _ingest(client: httpx.Client) -> None: - """Precondition helper: trigger ingestion. Raises on failure.""" - response = client.post("/assets/ingest") + response = client.post("/ingest") response.raise_for_status() def _list_assets(client: httpx.Client) -> dict: - """Precondition helper: fetch asset list. Raises on failure.""" response = client.get("/assets/") response.raise_for_status() return response.json() @@ -27,18 +26,14 @@ def _list_assets(client: httpx.Client) -> dict: @pytest.mark.e2e def test_ingest_and_list(http_client: httpx.Client): - """Trigger ingestion, verify count matches mock data, then list assets.""" - # Arrange mock_response = httpx.get(MOCK_INVENTORY_URL) mock_response.raise_for_status() expected_count = len(mock_response.json()) - # Act - response = http_client.post("/assets/ingest") + response = http_client.post("/ingest") - # Assert assert response.status_code == 200 - assert response.json()["ingested"] == expected_count + assert response.json()["assets_ingested"] == expected_count data = _list_assets(http_client) assert data["total"] == expected_count @@ -47,15 +42,11 @@ def test_ingest_and_list(http_client: httpx.Client): @pytest.mark.e2e def test_get_asset_by_id(http_client: httpx.Client): - """Retrieve a specific asset by its database ID.""" - # Arrange _ingest(http_client) asset_id = _list_assets(http_client)["assets"][0]["id"] - # Act response = http_client.get(f"/assets/{asset_id}") - # Assert assert response.status_code == 200 asset = response.json() assert asset["id"] == asset_id @@ -65,17 +56,13 @@ def test_get_asset_by_id(http_client: httpx.Client): @pytest.mark.e2e def test_get_asset_by_gold_source(http_client: httpx.Client): - """Retrieve an asset by its gold source type and ID.""" - # Arrange _ingest(http_client) first_asset = _list_assets(http_client)["assets"][0] source_type = first_asset["gold_source_type"] source_id = first_asset["gold_source_id"] - # Act response = http_client.get(f"/assets/by-gold-source/{source_type}/{source_id}") - # Assert assert response.status_code == 200 retrieved = response.json() assert retrieved["gold_source_id"] == source_id @@ -84,17 +71,13 @@ def test_get_asset_by_gold_source(http_client: httpx.Client): @pytest.mark.e2e def test_reingest_no_duplicates(http_client: httpx.Client): - """Re-ingesting should upsert, not duplicate.""" - # Arrange _ingest(http_client) total_after_first = _list_assets(http_client)["total"] - # Act - response = http_client.post("/assets/ingest") + response = http_client.post("/ingest") - # Assert assert response.status_code == 200 - assert response.json()["ingested"] == total_after_first + assert response.json()["assets_ingested"] == total_after_first assert _list_assets(http_client)["total"] == total_after_first diff --git a/verdict-backend/tests/e2e/test_cmdb_ingestion.py b/verdict-backend/tests/e2e/test_cmdb_ingestion.py new file mode 100644 index 0000000..8e93660 --- /dev/null +++ b/verdict-backend/tests/e2e/test_cmdb_ingestion.py @@ -0,0 +1,113 @@ +"""E2E tests for CMDB ingestion flow. + +These tests assume all services are running: +- PostgreSQL on localhost:5432 +- Mock Asset Inventory on localhost:4010 +- Mock CMDB on localhost:4011 +- Verdict API on localhost:8000 +""" + +import httpx +import pytest + +MOCK_CMDB_URL = "http://localhost:4011/systems" + + +def _ingest(client: httpx.Client) -> dict: + response = client.post("/ingest") + response.raise_for_status() + return response.json() + + +def _list_systems(client: httpx.Client) -> dict: + response = client.get("/systems/") + response.raise_for_status() + return response.json() + + +@pytest.mark.e2e +def test_ingest_and_list(http_client: httpx.Client): + mock_response = httpx.get(MOCK_CMDB_URL) + mock_response.raise_for_status() + expected_system_count = len(mock_response.json()) + + result = _ingest(http_client) + + assert result["systems_ingested"] == expected_system_count + + data = _list_systems(http_client) + assert data["total"] == expected_system_count + + +@pytest.mark.e2e +def test_get_system_by_id(http_client: httpx.Client): + _ingest(http_client) + system_id = _list_systems(http_client)["systems"][0]["id"] + + response = http_client.get(f"/systems/{system_id}") + + assert response.status_code == 200 + system = response.json() + assert system["id"] == system_id + assert "primary_fqdn" in system + + +@pytest.mark.e2e +def test_get_system_by_gold_source(http_client: httpx.Client): + _ingest(http_client) + first = _list_systems(http_client)["systems"][0] + + response = http_client.get( + f"/systems/by-gold-source/{first['gold_source_type']}/{first['gold_source_id']}" + ) + + assert response.status_code == 200 + assert response.json()["gold_source_id"] == first["gold_source_id"] + + +@pytest.mark.e2e +def test_system_includes_asset_ids(http_client: httpx.Client): + _ingest(http_client) + systems = _list_systems(http_client)["systems"] + system_with_assets = next(s for s in systems if s["asset_ids"]) + + assert len(system_with_assets["asset_ids"]) > 0 + + +@pytest.mark.e2e +def test_system_without_assets(http_client: httpx.Client): + _ingest(http_client) + systems = _list_systems(http_client)["systems"] + system_no_assets = next(s for s in systems if not s["asset_ids"]) + + assert system_no_assets["asset_ids"] == [] + + +@pytest.mark.e2e +def test_reingest_no_duplicates(http_client: httpx.Client): + _ingest(http_client) + total_after_first = _list_systems(http_client)["total"] + + _ingest(http_client) + + assert _list_systems(http_client)["total"] == total_after_first + + +@pytest.mark.e2e +def test_system_not_found(http_client: httpx.Client): + response = http_client.get("/systems/999999") + assert response.status_code == 404 + + +@pytest.mark.e2e +def test_gold_source_not_found(http_client: httpx.Client): + response = http_client.get("/systems/by-gold-source/cmdb/nonexistent") + assert response.status_code == 404 + + +@pytest.mark.e2e +def test_full_ingest_endpoint(http_client: httpx.Client): + result = _ingest(http_client) + + assert result["assets_ingested"] > 0 + assert result["systems_ingested"] > 0 diff --git a/verdict-backend/tests/factories/system.py b/verdict-backend/tests/factories/system.py new file mode 100644 index 0000000..c18c7d7 --- /dev/null +++ b/verdict-backend/tests/factories/system.py @@ -0,0 +1,16 @@ +# pyright: reportPrivateImportUsage=none +import factory + +from app.models.gold_source import GoldSourceType +from app.models.system import System +from tests.factories import BaseModelFactory + + +class SystemFactory(BaseModelFactory): + class Meta: # type: ignore[override] + model = System + + primary_fqdn = factory.Sequence(lambda n: f"host-{n:04d}.prod.example.com") + tags = factory.LazyFunction(lambda: ["env:production", "tier:backend"]) + gold_source_id = factory.Sequence(lambda n: f"SYS-{n:04d}") + gold_source_type = GoldSourceType.CMDB diff --git a/verdict-backend/tests/models/mixin_test_model.py b/verdict-backend/tests/models/mixin_test_model.py index 3416ec5..bf1dcc2 100644 --- a/verdict-backend/tests/models/mixin_test_model.py +++ b/verdict-backend/tests/models/mixin_test_model.py @@ -1,6 +1,6 @@ -from app.models.base import BaseModel, GoldSourceMixin, TagsMixin, TimestampMixin +from app.models.base import BaseModel, GoldSourceMixin, TagsMixin -class MixinTestModel(TimestampMixin, GoldSourceMixin, TagsMixin, BaseModel, table=True): +class MixinTestModel(GoldSourceMixin, TagsMixin, BaseModel, table=True): __tablename__ = "mixin_test_model" # type: ignore[assignment] # Pyright doesn't understand SQLAlchemy's declared_attr name: str = "" diff --git a/verdict-backend/tests/routes/test_assets.py b/verdict-backend/tests/routes/test_assets.py index b36d470..9c0d863 100644 --- a/verdict-backend/tests/routes/test_assets.py +++ b/verdict-backend/tests/routes/test_assets.py @@ -1,33 +1,7 @@ -from collections.abc import Generator - -import httpx -import respx from httpx import AsyncClient -from app.deps import get_http_client -from app.main import app from tests.factories.asset import AssetFactory -INDEX_PAYLOAD = [ - {"id": "SVC-001", "name": "Online Banking Portal"}, - {"id": "SVC-002", "name": "Customer CRM"}, -] - -DETAIL_PAYLOADS = { - "SVC-001": { - "id": "SVC-001", - "name": "Online Banking Portal", - "description": "Customer-facing online banking application", - "tags": ["protection-level:high", "business-unit:retail-banking"], - }, - "SVC-002": { - "id": "SVC-002", - "name": "Customer CRM", - "description": "Internal CRM system for customer relationships", - "tags": ["protection-level:medium", "business-unit:customer-service"], - }, -} - async def test_list_assets(app_client: AsyncClient, db_session): AssetFactory.create() @@ -87,23 +61,3 @@ async def test_get_asset_by_gold_source_not_found(app_client: AsyncClient, db_se response = await app_client.get("/assets/by-gold-source/asset-inventory/nonexistent") assert response.status_code == 404 - - -@respx.mock -async def test_ingest_endpoint(app_client: AsyncClient, db_session): - respx.get("http://localhost:4010/assets").respond(200, json=INDEX_PAYLOAD) - for item_id, payload in DETAIL_PAYLOADS.items(): - respx.get(f"http://localhost:4010/assets/{item_id}").respond(200, json=payload) - - def _mock_http_client() -> Generator[httpx.Client, None, None]: - with httpx.Client() as client: - yield client - - app.dependency_overrides[get_http_client] = _mock_http_client - - response = await app_client.post("/assets/ingest") - - assert response.status_code == 200 - assert response.json()["ingested"] == 2 - - app.dependency_overrides.pop(get_http_client, None) diff --git a/verdict-backend/tests/routes/test_ingestion.py b/verdict-backend/tests/routes/test_ingestion.py new file mode 100644 index 0000000..333be6f --- /dev/null +++ b/verdict-backend/tests/routes/test_ingestion.py @@ -0,0 +1,107 @@ +from collections.abc import Generator + +import httpx +import pytest +import respx +from httpx import AsyncClient +from sqlmodel import Session, select + +from app.deps import get_http_client +from app.main import app +from app.models.system import System + + +@pytest.fixture +def mock_http_client() -> Generator[None, None, None]: + def _override() -> Generator[httpx.Client, None, None]: + with httpx.Client() as client: + yield client + + app.dependency_overrides[get_http_client] = _override + yield + app.dependency_overrides.pop(get_http_client, None) + + +ASSET_INDEX = [ + {"id": "SVC-001", "name": "Online Banking Portal"}, +] +ASSET_DETAIL = { + "SVC-001": { + "id": "SVC-001", + "name": "Online Banking Portal", + "description": "Customer-facing online banking application", + "tags": ["protection-level:high"], + }, +} +SYSTEM_INDEX = [ + {"id": "SYS-001", "primary_fqdn": "host.example.com"}, +] +SYSTEM_DETAIL = { + "SYS-001": { + "id": "SYS-001", + "primary_fqdn": "host.example.com", + "asset_gold_source_ids": ["SVC-001"], + "tags": ["env:production"], + }, +} + + +def _mock_all_services() -> None: + respx.get("http://localhost:4010/assets").respond(200, json=ASSET_INDEX) + for item_id, payload in ASSET_DETAIL.items(): + respx.get(f"http://localhost:4010/assets/{item_id}").respond(200, json=payload) + respx.get("http://localhost:4011/systems").respond(200, json=SYSTEM_INDEX) + for item_id, payload in SYSTEM_DETAIL.items(): + respx.get(f"http://localhost:4011/systems/{item_id}").respond(200, json=payload) + + +@respx.mock +async def test_full_ingest_endpoint(app_client: AsyncClient, db_session, mock_http_client): + _mock_all_services() + + response = await app_client.post("/ingest") + + assert response.status_code == 200 + data = response.json() + assert data["assets_ingested"] == 1 + assert data["systems_ingested"] == 1 + + +def _mock_system_failure() -> None: + respx.get("http://localhost:4010/assets").respond(200, json=ASSET_INDEX) + for item_id, payload in ASSET_DETAIL.items(): + respx.get(f"http://localhost:4010/assets/{item_id}").respond(200, json=payload) + respx.get("http://localhost:4011/systems").respond(200, json=SYSTEM_INDEX) + # System detail references a nonexistent asset + respx.get("http://localhost:4011/systems/SYS-001").respond( + 200, + json={ + "id": "SYS-001", + "primary_fqdn": "host.example.com", + "asset_gold_source_ids": ["NONEXISTENT"], + "tags": [], + }, + ) + + +@respx.mock +async def test_full_ingest_returns_502_on_system_failure( + app_client: AsyncClient, db_session, mock_http_client +): + _mock_system_failure() + + response = await app_client.post("/ingest") + + assert response.status_code == 502 + + +@respx.mock +async def test_full_ingest_does_not_persist_on_system_failure( + app_client: AsyncClient, db_session: Session, mock_http_client +): + _mock_system_failure() + + await app_client.post("/ingest") + + systems = db_session.exec(select(System)).all() + assert len(systems) == 0 diff --git a/verdict-backend/tests/routes/test_systems.py b/verdict-backend/tests/routes/test_systems.py new file mode 100644 index 0000000..8536dfc --- /dev/null +++ b/verdict-backend/tests/routes/test_systems.py @@ -0,0 +1,79 @@ +import sqlalchemy as sa +from httpx import AsyncClient + +from app.models.system import asset_system +from tests.factories.asset import AssetFactory +from tests.factories.system import SystemFactory + + +async def test_list_systems(app_client: AsyncClient, db_session): + SystemFactory.create() + SystemFactory.create() + + response = await app_client.get("/systems/") + + assert response.status_code == 200 + data = response.json() + assert len(data["systems"]) == 2 + assert data["total"] == 2 + + +async def test_list_systems_pagination(app_client: AsyncClient, db_session): + SystemFactory.create() + SystemFactory.create() + SystemFactory.create() + + response = await app_client.get("/systems/?offset=1&limit=1") + + assert response.status_code == 200 + data = response.json() + assert len(data["systems"]) == 1 + assert data["total"] == 3 + + +async def test_get_system_by_id(app_client: AsyncClient, db_session): + system = SystemFactory.create() + + response = await app_client.get(f"/systems/{system.id}") + + assert response.status_code == 200 + data = response.json() + assert data["primary_fqdn"] == system.primary_fqdn + assert data["gold_source_id"] == system.gold_source_id + + +async def test_get_system_not_found(app_client: AsyncClient, db_session): + response = await app_client.get("/systems/99999") + + assert response.status_code == 404 + + +async def test_get_system_by_gold_source(app_client: AsyncClient, db_session): + system = SystemFactory.create() + + response = await app_client.get( + f"/systems/by-gold-source/{system.gold_source_type}/{system.gold_source_id}" + ) + + assert response.status_code == 200 + data = response.json() + assert data["primary_fqdn"] == system.primary_fqdn + + +async def test_get_system_by_gold_source_not_found(app_client: AsyncClient, db_session): + response = await app_client.get("/systems/by-gold-source/cmdb/nonexistent") + + assert response.status_code == 404 + + +async def test_get_system_includes_asset_ids(app_client: AsyncClient, db_session): + asset = AssetFactory.create() + system = SystemFactory.create() + db_session.execute(sa.insert(asset_system).values(asset_id=asset.id, system_id=system.id)) + db_session.flush() + + response = await app_client.get(f"/systems/{system.id}") + + assert response.status_code == 200 + data = response.json() + assert data["asset_ids"] == [asset.id] diff --git a/verdict-backend/tests/services/test_asset_ingestion.py b/verdict-backend/tests/services/test_asset_ingestion.py index 632af57..70fedc3 100644 --- a/verdict-backend/tests/services/test_asset_ingestion.py +++ b/verdict-backend/tests/services/test_asset_ingestion.py @@ -3,7 +3,7 @@ from sqlmodel import select from app.errors import FetchError, ValidationError -from app.models.asset import Asset +from app.models.asset import Asset, AssetCreate from app.models.gold_source import GoldSourceType from app.result import Err, Ok from app.schemas.external.asset_inventory import AssetDetail @@ -126,6 +126,7 @@ def test_to_asset(): ) asset = to_asset(detail) + assert isinstance(asset, AssetCreate) assert asset.name == "Online Banking Portal" assert asset.description == "Customer-facing online banking application" assert asset.tags == ["protection-level:high"] diff --git a/verdict-backend/tests/services/test_cmdb_ingestion.py b/verdict-backend/tests/services/test_cmdb_ingestion.py new file mode 100644 index 0000000..4d43d75 --- /dev/null +++ b/verdict-backend/tests/services/test_cmdb_ingestion.py @@ -0,0 +1,307 @@ +import httpx +import respx +import sqlalchemy as sa +from sqlmodel import select + +from app.errors import FetchError, ValidationError +from app.models.gold_source import GoldSourceType +from app.models.system import System, SystemCreate, asset_system +from app.result import Err, Ok +from app.schemas.external.cmdb import SystemDetail +from app.services.cmdb_ingestion import ( + fetch_detail, + fetch_index, + ingest_systems, + to_system, +) +from tests.factories.asset import AssetFactory + +FAKE_URL = "http://mock-cmdb/systems" + +INDEX_PAYLOAD = [ + {"id": "SYS-001", "primary_fqdn": "banking-web-01.prod.example.com"}, + {"id": "SYS-002", "primary_fqdn": "shared-db-01.prod.example.com"}, +] + +DETAIL_PAYLOADS = { + "SYS-001": { + "id": "SYS-001", + "primary_fqdn": "banking-web-01.prod.example.com", + "asset_gold_source_ids": ["SVC-001"], + "tags": ["env:production", "tier:frontend"], + }, + "SYS-002": { + "id": "SYS-002", + "primary_fqdn": "shared-db-01.prod.example.com", + "asset_gold_source_ids": ["SVC-001", "SVC-002"], + "tags": ["env:production", "tier:data"], + }, +} + + +def _mock_index_and_details() -> None: + respx.get(FAKE_URL).respond(200, json=INDEX_PAYLOAD) + for item_id, payload in DETAIL_PAYLOADS.items(): + respx.get(f"{FAKE_URL}/{item_id}").respond(200, json=payload) + + +# --- fetch_index tests --- + + +@respx.mock +def test_fetch_index_success(): + respx.get(FAKE_URL).respond(200, json=INDEX_PAYLOAD) + + with httpx.Client() as client: + result = fetch_index(client, FAKE_URL) + + assert isinstance(result, Ok) + assert len(result.value) == 2 + assert result.value[0].id == "SYS-001" + + +@respx.mock +def test_fetch_index_invalid_response(): + respx.get(FAKE_URL).respond(200, json=[{"name": "Missing fields"}]) + + with httpx.Client() as client: + result = fetch_index(client, FAKE_URL) + + assert isinstance(result, Err) + assert isinstance(result.value, ValidationError) + + +@respx.mock +def test_fetch_index_fetch_error(): + respx.get(FAKE_URL).respond(500) + + with httpx.Client() as client: + result = fetch_index(client, FAKE_URL) + + assert isinstance(result, Err) + assert isinstance(result.value, FetchError) + + +# --- fetch_detail tests --- + + +@respx.mock +def test_fetch_detail_success(): + respx.get(f"{FAKE_URL}/SYS-001").respond(200, json=DETAIL_PAYLOADS["SYS-001"]) + + with httpx.Client() as client: + result = fetch_detail(client, FAKE_URL, "SYS-001") + + assert isinstance(result, Ok) + assert result.value.id == "SYS-001" + assert result.value.primary_fqdn == "banking-web-01.prod.example.com" + assert result.value.asset_gold_source_ids == ["SVC-001"] + + +@respx.mock +def test_fetch_detail_not_found(): + respx.get(f"{FAKE_URL}/nonexistent").respond(404) + + with httpx.Client() as client: + result = fetch_detail(client, FAKE_URL, "nonexistent") + + assert isinstance(result, Err) + assert isinstance(result.value, FetchError) + + +@respx.mock +def test_fetch_detail_invalid_response(): + respx.get(f"{FAKE_URL}/SYS-001").respond(200, json={"id": "SYS-001"}) + + with httpx.Client() as client: + result = fetch_detail(client, FAKE_URL, "SYS-001") + + assert isinstance(result, Err) + assert isinstance(result.value, ValidationError) + + +# --- to_system tests --- + + +def test_to_system(): + detail = SystemDetail( + id="SYS-001", + primary_fqdn="banking-web-01.prod.example.com", + asset_gold_source_ids=["SVC-001"], + tags=["env:production"], + ) + system = to_system(detail) + + assert isinstance(system, SystemCreate) + assert system.primary_fqdn == "banking-web-01.prod.example.com" + assert system.gold_source_id == "SYS-001" + assert system.gold_source_type == GoldSourceType.CMDB + + +# --- ingest_systems integration tests --- + + +@respx.mock +def test_ingest_valid_response(db_session): + AssetFactory.create(gold_source_id="SVC-001") + AssetFactory.create(gold_source_id="SVC-002") + _mock_index_and_details() + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + + assert isinstance(result, Ok) + assert len(result.value) == 2 + + systems = list(db_session.exec(select(System).order_by(System.gold_source_id)).all()) + assert len(systems) == 2 + assert systems[0].primary_fqdn == "banking-web-01.prod.example.com" + + +@respx.mock +def test_ingest_system_without_asset_links(db_session): + index = [{"id": "SYS-003", "primary_fqdn": "monitoring-01.infra.example.com"}] + detail = { + "id": "SYS-003", + "primary_fqdn": "monitoring-01.infra.example.com", + "asset_gold_source_ids": [], + "tags": ["env:production"], + } + respx.get(FAKE_URL).respond(200, json=index) + respx.get(f"{FAKE_URL}/SYS-003").respond(200, json=detail) + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + + assert isinstance(result, Ok) + assert len(result.value) == 1 + + +@respx.mock +def test_ingest_unresolvable_asset_rejects_all(db_session): + index = [{"id": "SYS-001", "primary_fqdn": "host.example.com"}] + detail = { + "id": "SYS-001", + "primary_fqdn": "host.example.com", + "asset_gold_source_ids": ["NONEXISTENT"], + "tags": [], + } + respx.get(FAKE_URL).respond(200, json=index) + respx.get(f"{FAKE_URL}/SYS-001").respond(200, json=detail) + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + + assert isinstance(result, Err) + assert isinstance(result.value, ValidationError) + + systems = list(db_session.exec(select(System)).all()) + assert len(systems) == 0 + + +@respx.mock +def test_reingest_upserts_without_duplicates(db_session): + AssetFactory.create(gold_source_id="SVC-001") + AssetFactory.create(gold_source_id="SVC-002") + _mock_index_and_details() + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + assert isinstance(result, Ok) + db_session.flush() + + updated_details = { + k: {**v, "primary_fqdn": f"updated-{v['primary_fqdn']}"} for k, v in DETAIL_PAYLOADS.items() + } + respx.get(FAKE_URL).respond(200, json=INDEX_PAYLOAD) + for item_id, payload in updated_details.items(): + respx.get(f"{FAKE_URL}/{item_id}").respond(200, json=payload) + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + + assert isinstance(result, Ok) + + systems = list(db_session.exec(select(System)).all()) + assert len(systems) == 2 + fqdns = sorted(s.primary_fqdn for s in systems) + assert all(f.startswith("updated-") for f in fqdns) + + +@respx.mock +def test_reingest_updates_asset_links(db_session): + asset1 = AssetFactory.create(gold_source_id="SVC-001") + asset2 = AssetFactory.create(gold_source_id="SVC-002") + + index = [{"id": "SYS-001", "primary_fqdn": "host.example.com"}] + detail_v1 = { + "id": "SYS-001", + "primary_fqdn": "host.example.com", + "asset_gold_source_ids": ["SVC-001"], + "tags": [], + } + respx.get(FAKE_URL).respond(200, json=index) + respx.get(f"{FAKE_URL}/SYS-001").respond(200, json=detail_v1) + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + assert isinstance(result, Ok) + db_session.flush() + + system = db_session.exec(select(System)).first() + assert system is not None + link_ids = { + row.asset_id + for row in db_session.execute( + sa.select(asset_system.c.asset_id).where(asset_system.c.system_id == system.id) + ).all() + } + assert link_ids == {asset1.id} + + detail_v2 = { + "id": "SYS-001", + "primary_fqdn": "host.example.com", + "asset_gold_source_ids": ["SVC-002"], + "tags": [], + } + respx.get(FAKE_URL).respond(200, json=index) + respx.get(f"{FAKE_URL}/SYS-001").respond(200, json=detail_v2) + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + assert isinstance(result, Ok) + db_session.flush() + + link_ids = { + row.asset_id + for row in db_session.execute( + sa.select(asset_system.c.asset_id).where(asset_system.c.system_id == system.id) + ).all() + } + assert link_ids == {asset2.id} + + +@respx.mock +def test_ingest_fetch_error(db_session): + respx.get(FAKE_URL).respond(500) + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + + assert isinstance(result, Err) + assert isinstance(result.value, FetchError) + + +@respx.mock +def test_ingest_detail_fetch_error_persists_nothing(db_session): + respx.get(FAKE_URL).respond(200, json=INDEX_PAYLOAD) + respx.get(f"{FAKE_URL}/SYS-001").respond(500) + + with httpx.Client() as client: + result = ingest_systems(db_session, client, FAKE_URL) + + assert isinstance(result, Err) + assert isinstance(result.value, FetchError) + + systems = list(db_session.exec(select(System)).all()) + assert len(systems) == 0